From e6e4625ce286cba623b913e9d862eb83dbda61ef Mon Sep 17 00:00:00 2001 From: Aahlad Date: Thu, 17 Sep 2026 18:22:21 +0000 Subject: [PATCH] Upload results for ademamix golden --- configs/ademamix_golden.json | 272 ++++ .../criteo1tb_pytorch_09-14-2026-17-27-32.log | 521 +++++++ .../trial_1/eval_measurements.csv | 39 + ...t.tfevents.1789406963.daf5ed747178.38.0.v2 | Bin 0 -> 57482 bytes .../criteo1tb_pytorch/trial_1/flags_0.json | 62 + .../trial_1/measurements.csv | 156 ++ .../trial_1/meta_data_0.json | 72 + .../fastmri_pytorch_09-16-2026-06-26-24.log | 439 ++++++ .../trial_1/eval_measurements.csv | 16 + ...t.tfevents.1789540087.d4b46661c922.38.0.v2 | Bin 0 -> 39542 bytes .../fastmri_pytorch/trial_1/flags_0.json | 62 + .../fastmri_pytorch/trial_1/measurements.csv | 131 ++ .../fastmri_pytorch/trial_1/meta_data_0.json | 71 + ...ewebedu_lm_pytorch_09-11-2026-18-40-54.log | 470 ++++++ .../trial_1/eval_measurements.csv | 7 + ...t.tfevents.1789152077.1fd2a5398af3.38.0.v2 | Bin 0 -> 39308 bytes .../trial_1/flags_0.json | 62 + .../trial_1/measurements.csv | 170 +++ .../trial_1/meta_data_0.json | 69 + ...net_resnet_pytorch_09-12-2026-02-32-07.log | 1087 ++++++++++++++ .../trial_1/eval_measurements.csv | 39 + ...t.tfevents.1789180376.347fee244df2.38.0.v2 | Bin 0 -> 110245 bytes .../trial_1/flags_0.json | 63 + .../trial_1/measurements.csv | 371 +++++ .../trial_1/meta_data_0.json | 71 + ...agenet_vit_pytorch_09-12-2026-10-28-23.log | 401 ++++++ ...t.tfevents.1789208995.df1b65fa5772.38.0.v2 | Bin 0 -> 270 bytes .../imagenet_vit_pytorch/trial_1/flags_0.json | 63 + .../trial_1/measurements.csv | 2 + .../trial_1/meta_data_0.json | 74 + ..._conformer_pytorch_09-15-2026-04-25-35.log | 1011 +++++++++++++ .../trial_1/eval_measurements.csv | 32 + ...t.tfevents.1789446428.bc3590e26d6c.38.0.v2 | Bin 0 -> 84000 bytes .../trial_1/flags_0.json | 62 + .../trial_1/measurements.csv | 275 ++++ .../trial_1/meta_data_0.json | 71 + ...deepspeech_pytorch_09-16-2026-01-52-47.log | 1023 +++++++++++++ .../trial_1/eval_measurements.csv | 40 + ...t.tfevents.1789523586.7e9feaa71658.38.0.v2 | Bin 0 -> 93662 bytes .../trial_1/flags_0.json | 62 + .../trial_1/measurements.csv | 286 ++++ .../trial_1/meta_data_0.json | 77 + .../ogbg_pytorch_09-16-2026-07-32-45.log | 452 ++++++ .../trial_1/eval_measurements.csv | 12 + ...t.tfevents.1789543988.791f03d5f744.38.0.v2 | Bin 0 -> 43428 bytes .../study_0/ogbg_pytorch/trial_1/flags_0.json | 63 + .../ogbg_pytorch/trial_1/measurements.csv | 153 ++ .../ogbg_pytorch/trial_1/meta_data_0.json | 70 + .../wmt_pytorch/trial_1/eval_measurements.csv | 34 + ...t.tfevents.1789292648.b0019b52843f.38.0.v2 | Bin 0 -> 119180 bytes .../study_0/wmt_pytorch/trial_1/flags_0.json | 63 + .../wmt_pytorch/trial_1/measurements.csv | 403 ++++++ .../wmt_pytorch/trial_1/meta_data_0.json | 71 + .../wmt_pytorch_09-13-2026-09-42-13.log | 1132 +++++++++++++++ .../criteo1tb_pytorch_09-15-2026-19-27-27.log | 523 +++++++ .../trial_1/eval_measurements.csv | 39 + ...t.tfevents.1789500541.fe7c58f35b35.38.0.v2 | Bin 0 -> 57683 bytes .../criteo1tb_pytorch/trial_1/flags_0.json | 62 + .../trial_1/measurements.csv | 157 ++ .../trial_1/meta_data_0.json | 72 + .../fastmri_pytorch_09-12-2026-09-41-29.log | 443 ++++++ .../trial_1/eval_measurements.csv | 16 + ...t.tfevents.1789206186.5ce0f4169081.38.0.v2 | Bin 0 -> 39944 bytes .../fastmri_pytorch/trial_1/flags_0.json | 62 + .../fastmri_pytorch/trial_1/measurements.csv | 133 ++ .../fastmri_pytorch/trial_1/meta_data_0.json | 71 + ...ewebedu_lm_pytorch_09-11-2026-14-45-42.log | 470 ++++++ .../trial_1/eval_measurements.csv | 7 + ...t.tfevents.1789137963.1aa19998bf4d.38.0.v2 | Bin 0 -> 39308 bytes .../trial_1/flags_0.json | 62 + .../trial_1/measurements.csv | 170 +++ .../trial_1/meta_data_0.json | 69 + ...net_resnet_pytorch_09-12-2026-11-08-18.log | 1063 ++++++++++++++ .../trial_1/eval_measurements.csv | 39 + ...t.tfevents.1789211374.c9d641ec7cfd.38.0.v2 | Bin 0 -> 107782 bytes .../trial_1/flags_0.json | 63 + .../trial_1/measurements.csv | 359 +++++ .../trial_1/meta_data_0.json | 71 + ...agenet_vit_pytorch_09-13-2026-17-32-42.log | 1089 ++++++++++++++ .../trial_1/eval_measurements.csv | 39 + ...t.tfevents.1789320798.dfd4fda67ccf.38.0.v2 | Bin 0 -> 113728 bytes .../imagenet_vit_pytorch/trial_1/flags_0.json | 63 + .../trial_1/measurements.csv | 388 +++++ .../trial_1/meta_data_0.json | 74 + ..._conformer_pytorch_09-11-2026-14-43-03.log | 1192 +++++++++++++++ .../trial_1/eval_measurements.csv | 39 + ...t.tfevents.1789137799.bc2a5acb9a19.38.0.v2 | Bin 0 -> 100105 bytes .../trial_1/flags_0.json | 62 + .../trial_1/measurements.csv | 322 +++++ .../trial_1/meta_data_0.json | 71 + ...deepspeech_pytorch_09-14-2026-01-11-07.log | 1031 +++++++++++++ .../trial_1/eval_measurements.csv | 40 + ...t.tfevents.1789348370.925fa6ef3e4d.38.0.v2 | Bin 0 -> 93254 bytes .../trial_1/flags_0.json | 62 + .../trial_1/measurements.csv | 284 ++++ .../trial_1/meta_data_0.json | 77 + .../ogbg_pytorch_09-13-2026-01-07-00.log | 463 ++++++ .../trial_1/eval_measurements.csv | 13 + ...t.tfevents.1789261715.f68f7f208e51.38.0.v2 | Bin 0 -> 45446 bytes .../study_1/ogbg_pytorch/trial_1/flags_0.json | 63 + .../ogbg_pytorch/trial_1/measurements.csv | 157 ++ .../ogbg_pytorch/trial_1/meta_data_0.json | 70 + .../wmt_pytorch/trial_1/eval_measurements.csv | 39 + ...t.tfevents.1789549050.1803622dc733.37.0.v2 | Bin 0 -> 134694 bytes .../study_1/wmt_pytorch/trial_1/flags_0.json | 63 + .../wmt_pytorch/trial_1/measurements.csv | 451 ++++++ .../wmt_pytorch/trial_1/meta_data_0.json | 71 + .../wmt_pytorch_09-16-2026-08-57-11.log | 1258 ++++++++++++++++ .../criteo1tb_pytorch_09-14-2026-22-26-15.log | 523 +++++++ .../trial_1/eval_measurements.csv | 39 + ...t.tfevents.1789424879.6cdd716f9ad6.38.0.v2 | Bin 0 -> 57683 bytes .../criteo1tb_pytorch/trial_1/flags_0.json | 62 + .../trial_1/measurements.csv | 157 ++ .../trial_1/meta_data_0.json | 72 + .../fastmri_pytorch_09-16-2026-07-15-03.log | 402 ++++++ .../trial_1/eval_measurements.csv | 7 + ...t.tfevents.1789542963.37feccc55827.38.0.v2 | Bin 0 -> 30347 bytes .../fastmri_pytorch/trial_1/flags_0.json | 62 + .../fastmri_pytorch/trial_1/measurements.csv | 126 ++ .../fastmri_pytorch/trial_1/meta_data_0.json | 71 + ...ewebedu_lm_pytorch_09-11-2026-22-36-05.log | 470 ++++++ .../trial_1/eval_measurements.csv | 7 + ...t.tfevents.1789166185.b93e232cdf61.38.0.v2 | Bin 0 -> 39308 bytes .../trial_1/flags_0.json | 62 + .../trial_1/measurements.csv | 170 +++ .../trial_1/meta_data_0.json | 69 + ...net_resnet_pytorch_09-13-2026-02-40-09.log | 1027 +++++++++++++ .../trial_1/eval_measurements.csv | 39 + ...t.tfevents.1789267244.59c181d2b77d.38.0.v2 | Bin 0 -> 104125 bytes .../trial_1/flags_0.json | 63 + .../trial_1/measurements.csv | 341 +++++ .../trial_1/meta_data_0.json | 71 + ...agenet_vit_pytorch_09-15-2026-10-03-35.log | 1079 ++++++++++++++ .../trial_1/eval_measurements.csv | 39 + ...t.tfevents.1789466662.f86f86246ea5.38.0.v2 | Bin 0 -> 112693 bytes .../imagenet_vit_pytorch/trial_1/flags_0.json | 63 + .../trial_1/measurements.csv | 383 +++++ .../trial_1/meta_data_0.json | 74 + ..._conformer_pytorch_09-16-2026-18-33-54.log | 1186 +++++++++++++++ .../trial_1/eval_measurements.csv | 39 + ...t.tfevents.1789583649.d2c25cf1969c.38.0.v2 | Bin 0 -> 99697 bytes .../trial_1/flags_0.json | 62 + .../trial_1/measurements.csv | 320 +++++ .../trial_1/meta_data_0.json | 71 + ...deepspeech_pytorch_09-16-2026-18-08-25.log | 1035 ++++++++++++++ .../trial_1/eval_measurements.csv | 40 + ...t.tfevents.1789582202.2348acb64598.38.0.v2 | Bin 0 -> 94085 bytes .../trial_1/flags_0.json | 62 + .../trial_1/measurements.csv | 288 ++++ .../trial_1/meta_data_0.json | 77 + .../ogbg_pytorch_09-15-2026-07-17-58.log | 586 ++++++++ .../trial_1/eval_measurements.csv | 22 + ...t.tfevents.1789456786.d8fa15bfabc6.38.0.v2 | Bin 0 -> 66056 bytes .../study_2/ogbg_pytorch/trial_1/flags_0.json | 63 + .../ogbg_pytorch/trial_1/measurements.csv | 205 +++ .../ogbg_pytorch/trial_1/meta_data_0.json | 70 + .../wmt_pytorch/trial_1/eval_measurements.csv | 39 + ...t.tfevents.1789570917.3f807f090a40.38.0.v2 | Bin 0 -> 136140 bytes .../study_2/wmt_pytorch/trial_1/flags_0.json | 63 + .../wmt_pytorch/trial_1/measurements.csv | 458 ++++++ .../wmt_pytorch/trial_1/meta_data_0.json | 71 + .../wmt_pytorch_09-16-2026-15-00-16.log | 1272 +++++++++++++++++ 162 files changed, 33121 insertions(+) create mode 100644 configs/ademamix_golden.json create mode 100644 logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/criteo1tb_pytorch_09-14-2026-17-27-32.log create mode 100644 logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/trial_1/events.out.tfevents.1789406963.daf5ed747178.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_0/fastmri_pytorch/fastmri_pytorch_09-16-2026-06-26-24.log create mode 100644 logs/self_tuning/ademamix_golden/study_0/fastmri_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_0/fastmri_pytorch/trial_1/events.out.tfevents.1789540087.d4b46661c922.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_0/fastmri_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_0/fastmri_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_0/fastmri_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_0/finewebedu_lm_pytorch/finewebedu_lm_pytorch_09-11-2026-18-40-54.log create mode 100644 logs/self_tuning/ademamix_golden/study_0/finewebedu_lm_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_0/finewebedu_lm_pytorch/trial_1/events.out.tfevents.1789152077.1fd2a5398af3.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_0/finewebedu_lm_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_0/finewebedu_lm_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_0/finewebedu_lm_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_0/imagenet_resnet_pytorch/imagenet_resnet_pytorch_09-12-2026-02-32-07.log create mode 100644 logs/self_tuning/ademamix_golden/study_0/imagenet_resnet_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_0/imagenet_resnet_pytorch/trial_1/events.out.tfevents.1789180376.347fee244df2.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_0/imagenet_resnet_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_0/imagenet_resnet_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_0/imagenet_resnet_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_0/imagenet_vit_pytorch/imagenet_vit_pytorch_09-12-2026-10-28-23.log create mode 100644 logs/self_tuning/ademamix_golden/study_0/imagenet_vit_pytorch/trial_1/events.out.tfevents.1789208995.df1b65fa5772.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_0/imagenet_vit_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_0/imagenet_vit_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_0/imagenet_vit_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_0/librispeech_conformer_pytorch/librispeech_conformer_pytorch_09-15-2026-04-25-35.log create mode 100644 logs/self_tuning/ademamix_golden/study_0/librispeech_conformer_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_0/librispeech_conformer_pytorch/trial_1/events.out.tfevents.1789446428.bc3590e26d6c.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_0/librispeech_conformer_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_0/librispeech_conformer_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_0/librispeech_conformer_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_0/librispeech_deepspeech_pytorch/librispeech_deepspeech_pytorch_09-16-2026-01-52-47.log create mode 100644 logs/self_tuning/ademamix_golden/study_0/librispeech_deepspeech_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_0/librispeech_deepspeech_pytorch/trial_1/events.out.tfevents.1789523586.7e9feaa71658.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_0/librispeech_deepspeech_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_0/librispeech_deepspeech_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_0/librispeech_deepspeech_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/ogbg_pytorch_09-16-2026-07-32-45.log create mode 100644 logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/trial_1/events.out.tfevents.1789543988.791f03d5f744.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_0/wmt_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_0/wmt_pytorch/trial_1/events.out.tfevents.1789292648.b0019b52843f.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_0/wmt_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_0/wmt_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_0/wmt_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_0/wmt_pytorch/wmt_pytorch_09-13-2026-09-42-13.log create mode 100644 logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/criteo1tb_pytorch_09-15-2026-19-27-27.log create mode 100644 logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/trial_1/events.out.tfevents.1789500541.fe7c58f35b35.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/fastmri_pytorch_09-12-2026-09-41-29.log create mode 100644 logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/trial_1/events.out.tfevents.1789206186.5ce0f4169081.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_1/finewebedu_lm_pytorch/finewebedu_lm_pytorch_09-11-2026-14-45-42.log create mode 100644 logs/self_tuning/ademamix_golden/study_1/finewebedu_lm_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_1/finewebedu_lm_pytorch/trial_1/events.out.tfevents.1789137963.1aa19998bf4d.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_1/finewebedu_lm_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_1/finewebedu_lm_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_1/finewebedu_lm_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_1/imagenet_resnet_pytorch/imagenet_resnet_pytorch_09-12-2026-11-08-18.log create mode 100644 logs/self_tuning/ademamix_golden/study_1/imagenet_resnet_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_1/imagenet_resnet_pytorch/trial_1/events.out.tfevents.1789211374.c9d641ec7cfd.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_1/imagenet_resnet_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_1/imagenet_resnet_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_1/imagenet_resnet_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_1/imagenet_vit_pytorch/imagenet_vit_pytorch_09-13-2026-17-32-42.log create mode 100644 logs/self_tuning/ademamix_golden/study_1/imagenet_vit_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_1/imagenet_vit_pytorch/trial_1/events.out.tfevents.1789320798.dfd4fda67ccf.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_1/imagenet_vit_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_1/imagenet_vit_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_1/imagenet_vit_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_1/librispeech_conformer_pytorch/librispeech_conformer_pytorch_09-11-2026-14-43-03.log create mode 100644 logs/self_tuning/ademamix_golden/study_1/librispeech_conformer_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_1/librispeech_conformer_pytorch/trial_1/events.out.tfevents.1789137799.bc2a5acb9a19.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_1/librispeech_conformer_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_1/librispeech_conformer_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_1/librispeech_conformer_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/librispeech_deepspeech_pytorch_09-14-2026-01-11-07.log create mode 100644 logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/trial_1/events.out.tfevents.1789348370.925fa6ef3e4d.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/ogbg_pytorch_09-13-2026-01-07-00.log create mode 100644 logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/trial_1/events.out.tfevents.1789261715.f68f7f208e51.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/trial_1/events.out.tfevents.1789549050.1803622dc733.37.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/wmt_pytorch_09-16-2026-08-57-11.log create mode 100644 logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/criteo1tb_pytorch_09-14-2026-22-26-15.log create mode 100644 logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/trial_1/events.out.tfevents.1789424879.6cdd716f9ad6.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_2/fastmri_pytorch/fastmri_pytorch_09-16-2026-07-15-03.log create mode 100644 logs/self_tuning/ademamix_golden/study_2/fastmri_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_2/fastmri_pytorch/trial_1/events.out.tfevents.1789542963.37feccc55827.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_2/fastmri_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_2/fastmri_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_2/fastmri_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_2/finewebedu_lm_pytorch/finewebedu_lm_pytorch_09-11-2026-22-36-05.log create mode 100644 logs/self_tuning/ademamix_golden/study_2/finewebedu_lm_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_2/finewebedu_lm_pytorch/trial_1/events.out.tfevents.1789166185.b93e232cdf61.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_2/finewebedu_lm_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_2/finewebedu_lm_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_2/finewebedu_lm_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/imagenet_resnet_pytorch_09-13-2026-02-40-09.log create mode 100644 logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/trial_1/events.out.tfevents.1789267244.59c181d2b77d.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/imagenet_vit_pytorch_09-15-2026-10-03-35.log create mode 100644 logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/trial_1/events.out.tfevents.1789466662.f86f86246ea5.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/librispeech_conformer_pytorch_09-16-2026-18-33-54.log create mode 100644 logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/trial_1/events.out.tfevents.1789583649.d2c25cf1969c.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/librispeech_deepspeech_pytorch_09-16-2026-18-08-25.log create mode 100644 logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/trial_1/events.out.tfevents.1789582202.2348acb64598.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_2/ogbg_pytorch/ogbg_pytorch_09-15-2026-07-17-58.log create mode 100644 logs/self_tuning/ademamix_golden/study_2/ogbg_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_2/ogbg_pytorch/trial_1/events.out.tfevents.1789456786.d8fa15bfabc6.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_2/ogbg_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_2/ogbg_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_2/ogbg_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_2/wmt_pytorch/trial_1/eval_measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_2/wmt_pytorch/trial_1/events.out.tfevents.1789570917.3f807f090a40.38.0.v2 create mode 100644 logs/self_tuning/ademamix_golden/study_2/wmt_pytorch/trial_1/flags_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_2/wmt_pytorch/trial_1/measurements.csv create mode 100644 logs/self_tuning/ademamix_golden/study_2/wmt_pytorch/trial_1/meta_data_0.json create mode 100644 logs/self_tuning/ademamix_golden/study_2/wmt_pytorch/wmt_pytorch_09-16-2026-15-00-16.log diff --git a/configs/ademamix_golden.json b/configs/ademamix_golden.json new file mode 100644 index 00000000..7e99b881 --- /dev/null +++ b/configs/ademamix_golden.json @@ -0,0 +1,272 @@ +{ + "0": { + "framework": "pytorch", + "workload": "imagenet_resnet", + "dataset": "imagenet", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_0", + "rng_seed": -1820589156, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "1": { + "framework": "pytorch", + "workload": "imagenet_resnet", + "dataset": "imagenet", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_1", + "rng_seed": -1054496255, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "2": { + "framework": "pytorch", + "workload": "imagenet_resnet", + "dataset": "imagenet", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_2", + "rng_seed": -1587548617, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "3": { + "framework": "pytorch", + "workload": "imagenet_vit", + "dataset": "imagenet", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_0", + "rng_seed": 1076931326, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "4": { + "framework": "pytorch", + "workload": "imagenet_vit", + "dataset": "imagenet", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_1", + "rng_seed": 1612614542, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "5": { + "framework": "pytorch", + "workload": "imagenet_vit", + "dataset": "imagenet", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_2", + "rng_seed": -83011591, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "6": { + "framework": "pytorch", + "workload": "fastmri", + "dataset": "fastmri", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_0", + "rng_seed": -1941863332, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "7": { + "framework": "pytorch", + "workload": "fastmri", + "dataset": "fastmri", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_1", + "rng_seed": -638637756, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "8": { + "framework": "pytorch", + "workload": "fastmri", + "dataset": "fastmri", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_2", + "rng_seed": -482250772, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "9": { + "framework": "pytorch", + "workload": "ogbg", + "dataset": "ogbg", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_0", + "rng_seed": 1297901242, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "10": { + "framework": "pytorch", + "workload": "ogbg", + "dataset": "ogbg", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_1", + "rng_seed": -1419473739, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "11": { + "framework": "pytorch", + "workload": "ogbg", + "dataset": "ogbg", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_2", + "rng_seed": -777571328, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "12": { + "framework": "pytorch", + "workload": "wmt", + "dataset": "wmt", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_0", + "rng_seed": -964980308, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "13": { + "framework": "pytorch", + "workload": "wmt", + "dataset": "wmt", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_1", + "rng_seed": 45840584, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "14": { + "framework": "pytorch", + "workload": "wmt", + "dataset": "wmt", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_2", + "rng_seed": 16914702, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "15": { + "framework": "pytorch", + "workload": "librispeech_deepspeech", + "dataset": "librispeech", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_0", + "rng_seed": 1619867207, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "16": { + "framework": "pytorch", + "workload": "librispeech_deepspeech", + "dataset": "librispeech", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_1", + "rng_seed": 611108887, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "17": { + "framework": "pytorch", + "workload": "librispeech_deepspeech", + "dataset": "librispeech", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_2", + "rng_seed": -618110163, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "18": { + "framework": "pytorch", + "workload": "criteo1tb", + "dataset": "criteo1tb", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_0", + "rng_seed": -858678099, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "19": { + "framework": "pytorch", + "workload": "criteo1tb", + "dataset": "criteo1tb", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_1", + "rng_seed": 1213513687, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "20": { + "framework": "pytorch", + "workload": "criteo1tb", + "dataset": "criteo1tb", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_2", + "rng_seed": -826981189, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "21": { + "framework": "pytorch", + "workload": "librispeech_conformer", + "dataset": "librispeech", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_0", + "rng_seed": 1128445477, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "22": { + "framework": "pytorch", + "workload": "librispeech_conformer", + "dataset": "librispeech", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_1", + "rng_seed": -486301922, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "23": { + "framework": "pytorch", + "workload": "librispeech_conformer", + "dataset": "librispeech", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_2", + "rng_seed": -582177137, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "24": { + "framework": "pytorch", + "workload": "finewebedu_lm", + "dataset": "fineweb_edu_10B", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_0", + "rng_seed": -235185075, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "25": { + "framework": "pytorch", + "workload": "finewebedu_lm", + "dataset": "fineweb_edu_10B", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_1", + "rng_seed": 140606013, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + }, + "26": { + "framework": "pytorch", + "workload": "finewebedu_lm", + "dataset": "fineweb_edu_10B", + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "experiment_dir": "submissions_a100/ademamix_golden/study_2", + "rng_seed": -965294040, + "tuning_ruleset": "self", + "num_tuning_trials": 1 + } +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/criteo1tb_pytorch_09-14-2026-17-27-32.log b/logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/criteo1tb_pytorch_09-14-2026-17-27-32.log new file mode 100644 index 00000000..3a6fdac5 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/criteo1tb_pytorch_09-14-2026-17-27-32.log @@ -0,0 +1,521 @@ +torchrun --redirects 1:0,2:0,3:0 --standalone --nnodes=1 --nproc_per_node=4 submission_runner.py --framework=pytorch --workload=criteo1tb --submission_path=submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py --data_dir=/data/criteo1tb --experiment_dir=/experiment_runs --experiment_name=submissions_a100/ademamix_golden/study_0 --overwrite=True --save_checkpoints=False --rng_seed=-858678099 --torch_compile=true --tuning_ruleset=self 2>&1 | tee -a /logs/criteo1tb_pytorch_09-14-2026-17-27-32.log +W0914 17:28:02.375000 9 site-packages/torch/distributed/run.py:803] +W0914 17:28:02.375000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0914 17:28:02.375000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0914 17:28:02.375000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-14 17:28:18.233756: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-14 17:28:18.233757: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-14 17:28:18.233768: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-14 17:28:18.233757: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789406898.885564 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789406898.885593 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789406898.885576 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789406898.885577 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789406898.963290 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789406898.963297 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789406898.963297 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789406898.963316 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789406900.625768 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789406900.625769 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789406900.625775 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789406900.625780 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789406900.625811 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789406900.625811 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789406900.625811 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789406900.625812 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789406900.625814 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789406900.625814 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789406900.625815 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789406900.625815 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789406900.625816 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789406900.625817 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789406900.625818 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789406900.625818 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789406933.228746 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789406933.228775 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789406933.228796 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789406933.228799 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W914 17:29:03.912545928 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0914 17:29:06.441272 139698671826112 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/criteo1tb_pytorch. +I0914 17:29:06.441272 140109222773952 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/criteo1tb_pytorch. +I0914 17:29:06.441272 139915216819392 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/criteo1tb_pytorch. +I0914 17:29:06.441285 140069821084864 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/criteo1tb_pytorch. +I0914 17:29:06.704234 140069821084864 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/criteo1tb_pytorch/trial_1. +I0914 17:29:06.975733 140069821084864 submission_runner.py:242] Initializing dataset. +I0914 17:29:06.975917 140069821084864 submission_runner.py:251] Initializing model. +W0914 17:29:22.244227 139698671826112 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0914 17:29:22.244227 140109222773952 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0914 17:29:22.244233 139915216819392 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0914 17:29:22.244280 140069821084864 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +I0914 17:29:22.244459 140069821084864 submission_runner.py:294] Initializing optimizer. +I0914 17:29:22.244872 140069821084864 submission_runner.py:299] Initializing metrics bundle. +I0914 17:29:22.245053 140069821084864 submission_runner.py:321] Initializing checkpoint and logger. +I0914 17:29:22.248365 140069821084864 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_0/criteo1tb_pytorch/trial_1/meta_data_0.json. +I0914 17:29:22.248414 139915216819392 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0914 17:29:22.248425 139698671826112 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0914 17:29:22.248546 139915216819392 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0914 17:29:22.248555 139698671826112 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0914 17:29:22.248480 140109222773952 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0914 17:29:22.248599 140069821084864 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0914 17:29:22.248627 140109222773952 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0914 17:29:22.248669 140069821084864 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0914 17:29:22.875969 140069821084864 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_0/criteo1tb_pytorch/trial_1/flags_0.json. +I0914 17:29:23.141260 140069821084864 submission_runner.py:359] Starting training loop. +I0914 17:29:34.354318 140046488897280 logging_writer.py:48] [0] global_step=0, grad_norm=0.5, loss=0.689198 +I0914 17:29:34.568252 140069821084864 submission.py:307] 0) loss = 0.689, grad_norm = 0.500 +I0914 17:29:34.896683 140069821084864 spec.py:333] Evaluating on the training split. +I0914 17:41:14.417857 140069821084864 spec.py:346] Evaluating on the validation split. +I0914 17:52:30.343646 140069821084864 spec.py:363] Evaluating on the test split. +I0914 18:05:19.399240 140069821084864 submission_runner.py:516] Time since start: 2156.26s, Step: 1, {'train/loss': 0.6890505460885573, 'validation/loss': 0.6889704567631297, 'validation/num_examples': 83274637, 'test/loss': 0.6891419952508223, 'test/num_examples': 95000000, 'score': 11.428372621536255, 'total_duration': 2156.258265018463, 'accumulated_submission_time': 11.428372621536255, 'accumulated_eval_time': 2144.502624988556, 'accumulated_logging_time': 0} +I0914 18:05:19.567724 140026536982272 logging_writer.py:48] [1] accumulated_eval_time=2144.5, accumulated_logging_time=0, accumulated_submission_time=11.4284, global_step=1, preemption_count=0, score=11.4284, test/loss=0.689142, test/num_examples=95000000, total_duration=2156.26, train/loss=0.689051, validation/loss=0.68897, validation/num_examples=83274637 +I0914 18:05:20.179536 140026528589568 logging_writer.py:48] [1] global_step=1, grad_norm=0.5, loss=0.68918 +I0914 18:05:20.182669 140069821084864 submission.py:307] 1) loss = 0.689, grad_norm = 0.500 +I0914 18:05:20.418234 140026536982272 logging_writer.py:48] [2] global_step=2, grad_norm=0.5, loss=0.683746 +I0914 18:05:20.421381 140069821084864 submission.py:307] 2) loss = 0.684, grad_norm = 0.500 +I0914 18:05:20.655732 140026528589568 logging_writer.py:48] [3] global_step=3, grad_norm=0.5, loss=0.672947 +I0914 18:05:20.658742 140069821084864 submission.py:307] 3) loss = 0.673, grad_norm = 0.500 +I0914 18:05:20.892677 140026536982272 logging_writer.py:48] [4] global_step=4, grad_norm=0.5, loss=0.656802 +I0914 18:05:20.895604 140069821084864 submission.py:307] 4) loss = 0.657, grad_norm = 0.500 +I0914 18:05:21.129808 140026528589568 logging_writer.py:48] [5] global_step=5, grad_norm=0.5, loss=0.635536 +I0914 18:05:21.132754 140069821084864 submission.py:307] 5) loss = 0.636, grad_norm = 0.500 +I0914 18:05:21.367295 140026536982272 logging_writer.py:48] [6] global_step=6, grad_norm=0.5, loss=0.6098 +I0914 18:05:21.370319 140069821084864 submission.py:307] 6) loss = 0.610, grad_norm = 0.500 +I0914 18:05:21.603744 140026528589568 logging_writer.py:48] [7] global_step=7, grad_norm=0.5, loss=0.579884 +I0914 18:05:21.606734 140069821084864 submission.py:307] 7) loss = 0.580, grad_norm = 0.500 +I0914 18:05:21.841030 140026536982272 logging_writer.py:48] [8] global_step=8, grad_norm=0.5, loss=0.546684 +I0914 18:05:21.844105 140069821084864 submission.py:307] 8) loss = 0.547, grad_norm = 0.500 +I0914 18:05:22.077088 140026528589568 logging_writer.py:48] [9] global_step=9, grad_norm=0.5, loss=0.510027 +I0914 18:05:22.080109 140069821084864 submission.py:307] 9) loss = 0.510, grad_norm = 0.500 +I0914 18:05:22.314311 140026536982272 logging_writer.py:48] [10] global_step=10, grad_norm=0.5, loss=0.472204 +I0914 18:05:22.317299 140069821084864 submission.py:307] 10) loss = 0.472, grad_norm = 0.500 +I0914 18:05:22.551140 140026528589568 logging_writer.py:48] [11] global_step=11, grad_norm=0.5, loss=0.432559 +I0914 18:05:22.554266 140069821084864 submission.py:307] 11) loss = 0.433, grad_norm = 0.500 +I0914 18:05:22.789183 140026536982272 logging_writer.py:48] [12] global_step=12, grad_norm=0.5, loss=0.393042 +I0914 18:05:22.792241 140069821084864 submission.py:307] 12) loss = 0.393, grad_norm = 0.500 +I0914 18:05:23.027754 140026528589568 logging_writer.py:48] [13] global_step=13, grad_norm=0.5, loss=0.353394 +I0914 18:05:23.030966 140069821084864 submission.py:307] 13) loss = 0.353, grad_norm = 0.500 +I0914 18:05:23.268493 140026536982272 logging_writer.py:48] [14] global_step=14, grad_norm=0.5, loss=0.315401 +I0914 18:05:23.271552 140069821084864 submission.py:307] 14) loss = 0.315, grad_norm = 0.500 +I0914 18:05:23.505109 140026528589568 logging_writer.py:48] [15] global_step=15, grad_norm=0.5, loss=0.279919 +I0914 18:05:23.508071 140069821084864 submission.py:307] 15) loss = 0.280, grad_norm = 0.500 +I0914 18:05:23.744445 140026536982272 logging_writer.py:48] [16] global_step=16, grad_norm=0.5, loss=0.250032 +I0914 18:05:23.747413 140069821084864 submission.py:307] 16) loss = 0.250, grad_norm = 0.500 +I0914 18:05:23.981341 140026528589568 logging_writer.py:48] [17] global_step=17, grad_norm=0.5, loss=0.224006 +I0914 18:05:23.984382 140069821084864 submission.py:307] 17) loss = 0.224, grad_norm = 0.500 +I0914 18:05:24.218593 140026536982272 logging_writer.py:48] [18] global_step=18, grad_norm=0.5, loss=0.202048 +I0914 18:05:24.221670 140069821084864 submission.py:307] 18) loss = 0.202, grad_norm = 0.500 +I0914 18:05:24.456483 140026528589568 logging_writer.py:48] [19] global_step=19, grad_norm=0.499999, loss=0.18201 +I0914 18:05:24.459590 140069821084864 submission.py:307] 19) loss = 0.182, grad_norm = 0.500 +I0914 18:05:24.693984 140026536982272 logging_writer.py:48] [20] global_step=20, grad_norm=0.499999, loss=0.169964 +I0914 18:05:24.697455 140069821084864 submission.py:307] 20) loss = 0.170, grad_norm = 0.500 +I0914 18:05:24.932443 140026528589568 logging_writer.py:48] [21] global_step=21, grad_norm=0.244878, loss=0.162229 +I0914 18:05:24.935459 140069821084864 submission.py:307] 21) loss = 0.162, grad_norm = 0.245 +I0914 18:05:25.170366 140026536982272 logging_writer.py:48] [22] global_step=22, grad_norm=0.177896, loss=0.157853 +I0914 18:05:25.173441 140069821084864 submission.py:307] 22) loss = 0.158, grad_norm = 0.178 +I0914 18:05:25.407094 140026528589568 logging_writer.py:48] [23] global_step=23, grad_norm=0.34179, loss=0.160895 +I0914 18:05:25.410216 140069821084864 submission.py:307] 23) loss = 0.161, grad_norm = 0.342 +I0914 18:05:25.645207 140026536982272 logging_writer.py:48] [24] global_step=24, grad_norm=0.448387, loss=0.160882 +I0914 18:05:25.648252 140069821084864 submission.py:307] 24) loss = 0.161, grad_norm = 0.448 +I0914 18:05:25.883526 140026528589568 logging_writer.py:48] [25] global_step=25, grad_norm=0.492796, loss=0.161316 +I0914 18:05:25.886588 140069821084864 submission.py:307] 25) loss = 0.161, grad_norm = 0.493 +I0914 18:05:26.120071 140026536982272 logging_writer.py:48] [26] global_step=26, grad_norm=0.484462, loss=0.161149 +I0914 18:05:26.124019 140069821084864 submission.py:307] 26) loss = 0.161, grad_norm = 0.484 +I0914 18:05:26.362023 140026528589568 logging_writer.py:48] [27] global_step=27, grad_norm=0.397565, loss=0.157649 +I0914 18:05:26.365041 140069821084864 submission.py:307] 27) loss = 0.158, grad_norm = 0.398 +I0914 18:05:26.600031 140026536982272 logging_writer.py:48] [28] global_step=28, grad_norm=0.263544, loss=0.15165 +I0914 18:05:26.603115 140069821084864 submission.py:307] 28) loss = 0.152, grad_norm = 0.264 +I0914 18:05:27.786464 140026528589568 logging_writer.py:48] [29] global_step=29, grad_norm=0.146678, loss=0.150101 +I0914 18:05:27.789508 140069821084864 submission.py:307] 29) loss = 0.150, grad_norm = 0.147 +I0914 18:05:30.188284 140026536982272 logging_writer.py:48] [30] global_step=30, grad_norm=0.106599, loss=0.148794 +I0914 18:05:30.191323 140069821084864 submission.py:307] 30) loss = 0.149, grad_norm = 0.107 +I0914 18:05:32.522961 140026528589568 logging_writer.py:48] [31] global_step=31, grad_norm=0.153016, loss=0.150432 +I0914 18:05:32.526018 140069821084864 submission.py:307] 31) loss = 0.150, grad_norm = 0.153 +I0914 18:05:34.506201 140026536982272 logging_writer.py:48] [32] global_step=32, grad_norm=0.186301, loss=0.148584 +I0914 18:05:34.509258 140069821084864 submission.py:307] 32) loss = 0.149, grad_norm = 0.186 +I0914 18:05:36.575149 140026528589568 logging_writer.py:48] [33] global_step=33, grad_norm=0.142947, loss=0.147012 +I0914 18:05:36.578276 140069821084864 submission.py:307] 33) loss = 0.147, grad_norm = 0.143 +I0914 18:05:38.588297 140026536982272 logging_writer.py:48] [34] global_step=34, grad_norm=0.0864599, loss=0.144175 +I0914 18:05:38.591542 140069821084864 submission.py:307] 34) loss = 0.144, grad_norm = 0.086 +I0914 18:05:40.815914 140026528589568 logging_writer.py:48] [35] global_step=35, grad_norm=0.108623, loss=0.143851 +I0914 18:05:40.819048 140069821084864 submission.py:307] 35) loss = 0.144, grad_norm = 0.109 +I0914 18:05:43.091388 140026536982272 logging_writer.py:48] [36] global_step=36, grad_norm=0.174385, loss=0.14486 +I0914 18:05:43.094461 140069821084864 submission.py:307] 36) loss = 0.145, grad_norm = 0.174 +I0914 18:05:45.539047 140026528589568 logging_writer.py:48] [37] global_step=37, grad_norm=0.156668, loss=0.142576 +I0914 18:05:45.542137 140069821084864 submission.py:307] 37) loss = 0.143, grad_norm = 0.157 +I0914 18:05:47.652343 140026536982272 logging_writer.py:48] [38] global_step=38, grad_norm=0.125792, loss=0.14751 +I0914 18:05:47.655410 140069821084864 submission.py:307] 38) loss = 0.148, grad_norm = 0.126 +I0914 18:05:49.992342 140026528589568 logging_writer.py:48] [39] global_step=39, grad_norm=0.0599951, loss=0.147111 +I0914 18:05:49.995478 140069821084864 submission.py:307] 39) loss = 0.147, grad_norm = 0.060 +I0914 18:05:52.103250 140026536982272 logging_writer.py:48] [40] global_step=40, grad_norm=0.0810338, loss=0.14846 +I0914 18:05:52.106470 140069821084864 submission.py:307] 40) loss = 0.148, grad_norm = 0.081 +I0914 18:05:54.544105 140026528589568 logging_writer.py:48] [41] global_step=41, grad_norm=0.0790589, loss=0.147586 +I0914 18:05:54.547455 140069821084864 submission.py:307] 41) loss = 0.148, grad_norm = 0.079 +I0914 18:05:57.012162 140026536982272 logging_writer.py:48] [42] global_step=42, grad_norm=0.0467737, loss=0.14555 +I0914 18:05:57.015231 140069821084864 submission.py:307] 42) loss = 0.146, grad_norm = 0.047 +I0914 18:05:59.008241 140026528589568 logging_writer.py:48] [43] global_step=43, grad_norm=0.0906159, loss=0.145067 +I0914 18:05:59.011291 140069821084864 submission.py:307] 43) loss = 0.145, grad_norm = 0.091 +I0914 18:06:01.233439 140026536982272 logging_writer.py:48] [44] global_step=44, grad_norm=0.0859875, loss=0.145604 +I0914 18:06:01.236559 140069821084864 submission.py:307] 44) loss = 0.146, grad_norm = 0.086 +I0914 18:06:03.502919 140026528589568 logging_writer.py:48] [45] global_step=45, grad_norm=0.039192, loss=0.146387 +I0914 18:06:03.506353 140069821084864 submission.py:307] 45) loss = 0.146, grad_norm = 0.039 +I0914 18:06:05.654223 140026536982272 logging_writer.py:48] [46] global_step=46, grad_norm=0.0361098, loss=0.148882 +I0914 18:06:05.657277 140069821084864 submission.py:307] 46) loss = 0.149, grad_norm = 0.036 +I0914 18:06:07.760335 140026528589568 logging_writer.py:48] [47] global_step=47, grad_norm=0.138199, loss=0.138495 +I0914 18:06:07.763506 140069821084864 submission.py:307] 47) loss = 0.138, grad_norm = 0.138 +I0914 18:06:09.598969 140026536982272 logging_writer.py:48] [48] global_step=48, grad_norm=0.0569327, loss=0.139154 +I0914 18:06:09.602086 140069821084864 submission.py:307] 48) loss = 0.139, grad_norm = 0.057 +I0914 18:06:11.673328 140026528589568 logging_writer.py:48] [49] global_step=49, grad_norm=0.125951, loss=0.13838 +I0914 18:06:11.676384 140069821084864 submission.py:307] 49) loss = 0.138, grad_norm = 0.126 +I0914 18:06:13.604892 140026536982272 logging_writer.py:48] [50] global_step=50, grad_norm=0.0325923, loss=0.13695 +I0914 18:06:13.607927 140069821084864 submission.py:307] 50) loss = 0.137, grad_norm = 0.033 +I0914 18:06:16.225606 140026528589568 logging_writer.py:48] [51] global_step=51, grad_norm=0.0937807, loss=0.136362 +I0914 18:06:16.228667 140069821084864 submission.py:307] 51) loss = 0.136, grad_norm = 0.094 +I0914 18:06:18.409425 140026536982272 logging_writer.py:48] [52] global_step=52, grad_norm=0.0366948, loss=0.136071 +I0914 18:06:18.412595 140069821084864 submission.py:307] 52) loss = 0.136, grad_norm = 0.037 +I0914 18:06:20.962333 140026528589568 logging_writer.py:48] [53] global_step=53, grad_norm=0.0915322, loss=0.136444 +I0914 18:06:20.965382 140069821084864 submission.py:307] 53) loss = 0.136, grad_norm = 0.092 +I0914 18:06:23.233685 140026536982272 logging_writer.py:48] [54] global_step=54, grad_norm=0.0268382, loss=0.134762 +I0914 18:06:23.236727 140069821084864 submission.py:307] 54) loss = 0.135, grad_norm = 0.027 +I0914 18:06:25.777383 140026528589568 logging_writer.py:48] [55] global_step=55, grad_norm=0.0731413, loss=0.133949 +I0914 18:06:25.780358 140069821084864 submission.py:307] 55) loss = 0.134, grad_norm = 0.073 +I0914 18:06:28.125356 140026536982272 logging_writer.py:48] [56] global_step=56, grad_norm=0.0211235, loss=0.134116 +I0914 18:06:28.128434 140069821084864 submission.py:307] 56) loss = 0.134, grad_norm = 0.021 +I0914 18:06:30.305749 140026528589568 logging_writer.py:48] [57] global_step=57, grad_norm=0.0403683, loss=0.132375 +I0914 18:06:30.308851 140069821084864 submission.py:307] 57) loss = 0.132, grad_norm = 0.040 +I0914 18:06:32.603077 140026536982272 logging_writer.py:48] [58] global_step=58, grad_norm=0.031725, loss=0.13557 +I0914 18:06:32.606173 140069821084864 submission.py:307] 58) loss = 0.136, grad_norm = 0.032 +I0914 18:06:34.925987 140026528589568 logging_writer.py:48] [59] global_step=59, grad_norm=0.0863037, loss=0.135108 +I0914 18:06:34.928972 140069821084864 submission.py:307] 59) loss = 0.135, grad_norm = 0.086 +I0914 18:06:37.314468 140026536982272 logging_writer.py:48] [60] global_step=60, grad_norm=0.0498793, loss=0.134947 +I0914 18:06:37.317532 140069821084864 submission.py:307] 60) loss = 0.135, grad_norm = 0.050 +I0914 18:06:39.601601 140026528589568 logging_writer.py:48] [61] global_step=61, grad_norm=0.0251996, loss=0.133363 +I0914 18:06:39.604614 140069821084864 submission.py:307] 61) loss = 0.133, grad_norm = 0.025 +I0914 18:06:42.060822 140026536982272 logging_writer.py:48] [62] global_step=62, grad_norm=0.0713536, loss=0.131519 +I0914 18:06:42.063892 140069821084864 submission.py:307] 62) loss = 0.132, grad_norm = 0.071 +I0914 18:06:44.297487 140026528589568 logging_writer.py:48] [63] global_step=63, grad_norm=0.102803, loss=0.13341 +I0914 18:06:44.300577 140069821084864 submission.py:307] 63) loss = 0.133, grad_norm = 0.103 +I0914 18:06:46.787613 140026536982272 logging_writer.py:48] [64] global_step=64, grad_norm=0.0958869, loss=0.131222 +I0914 18:06:46.790669 140069821084864 submission.py:307] 64) loss = 0.131, grad_norm = 0.096 +I0914 18:06:48.820811 140026528589568 logging_writer.py:48] [65] global_step=65, grad_norm=0.0291765, loss=0.131347 +I0914 18:06:48.823860 140069821084864 submission.py:307] 65) loss = 0.131, grad_norm = 0.029 +I0914 18:06:51.176029 140026536982272 logging_writer.py:48] [66] global_step=66, grad_norm=0.0735644, loss=0.135467 +I0914 18:06:51.179071 140069821084864 submission.py:307] 66) loss = 0.135, grad_norm = 0.074 +I0914 18:06:53.496782 140026528589568 logging_writer.py:48] [67] global_step=67, grad_norm=0.139301, loss=0.136478 +I0914 18:06:53.499833 140069821084864 submission.py:307] 67) loss = 0.136, grad_norm = 0.139 +I0914 18:06:55.903070 140026536982272 logging_writer.py:48] [68] global_step=68, grad_norm=0.132343, loss=0.134957 +I0914 18:06:55.906261 140069821084864 submission.py:307] 68) loss = 0.135, grad_norm = 0.132 +I0914 18:06:57.880855 140026528589568 logging_writer.py:48] [69] global_step=69, grad_norm=0.066005, loss=0.134286 +I0914 18:06:57.884340 140069821084864 submission.py:307] 69) loss = 0.134, grad_norm = 0.066 +I0914 18:07:00.102370 140026536982272 logging_writer.py:48] [70] global_step=70, grad_norm=0.0209384, loss=0.133619 +I0914 18:07:00.105420 140069821084864 submission.py:307] 70) loss = 0.134, grad_norm = 0.021 +I0914 18:07:02.606700 140026528589568 logging_writer.py:48] [71] global_step=71, grad_norm=0.0994184, loss=0.134716 +I0914 18:07:02.609796 140069821084864 submission.py:307] 71) loss = 0.135, grad_norm = 0.099 +I0914 18:07:04.791208 140026536982272 logging_writer.py:48] [72] global_step=72, grad_norm=0.188206, loss=0.134578 +I0914 18:07:04.794299 140069821084864 submission.py:307] 72) loss = 0.135, grad_norm = 0.188 +I0914 18:07:06.748131 140026528589568 logging_writer.py:48] [73] global_step=73, grad_norm=0.267239, loss=0.135906 +I0914 18:07:06.751164 140069821084864 submission.py:307] 73) loss = 0.136, grad_norm = 0.267 +I0914 18:07:09.084159 140026536982272 logging_writer.py:48] [74] global_step=74, grad_norm=0.298844, loss=0.133768 +I0914 18:07:09.087211 140069821084864 submission.py:307] 74) loss = 0.134, grad_norm = 0.299 +I0914 18:07:11.523450 140026528589568 logging_writer.py:48] [75] global_step=75, grad_norm=0.218726, loss=0.13344 +I0914 18:07:11.526650 140069821084864 submission.py:307] 75) loss = 0.133, grad_norm = 0.219 +I0914 18:07:13.649735 140026536982272 logging_writer.py:48] [76] global_step=76, grad_norm=0.0166625, loss=0.136167 +I0914 18:07:13.652859 140069821084864 submission.py:307] 76) loss = 0.136, grad_norm = 0.017 +I0914 18:07:16.289978 140026528589568 logging_writer.py:48] [77] global_step=77, grad_norm=0.251331, loss=0.135747 +I0914 18:07:16.293074 140069821084864 submission.py:307] 77) loss = 0.136, grad_norm = 0.251 +I0914 18:07:18.482015 140026536982272 logging_writer.py:48] [78] global_step=78, grad_norm=0.330274, loss=0.135295 +I0914 18:07:18.485179 140069821084864 submission.py:307] 78) loss = 0.135, grad_norm = 0.330 +I0914 18:07:20.718177 140026528589568 logging_writer.py:48] [79] global_step=79, grad_norm=0.201455, loss=0.133058 +I0914 18:07:20.721240 140069821084864 submission.py:307] 79) loss = 0.133, grad_norm = 0.201 +I0914 18:07:22.993709 140026536982272 logging_writer.py:48] [80] global_step=80, grad_norm=0.018247, loss=0.132844 +I0914 18:07:22.996711 140069821084864 submission.py:307] 80) loss = 0.133, grad_norm = 0.018 +I0914 18:07:25.204940 140026528589568 logging_writer.py:48] [81] global_step=81, grad_norm=0.180408, loss=0.134019 +I0914 18:07:25.207946 140069821084864 submission.py:307] 81) loss = 0.134, grad_norm = 0.180 +I0914 18:07:27.407818 140026536982272 logging_writer.py:48] [82] global_step=82, grad_norm=0.2469, loss=0.133622 +I0914 18:07:27.410876 140069821084864 submission.py:307] 82) loss = 0.134, grad_norm = 0.247 +I0914 18:07:29.914128 140026528589568 logging_writer.py:48] [83] global_step=83, grad_norm=0.194889, loss=0.12949 +I0914 18:07:29.917112 140069821084864 submission.py:307] 83) loss = 0.129, grad_norm = 0.195 +I0914 18:07:32.050786 140026536982272 logging_writer.py:48] [84] global_step=84, grad_norm=0.0518726, loss=0.129946 +I0914 18:07:32.053917 140069821084864 submission.py:307] 84) loss = 0.130, grad_norm = 0.052 +I0914 18:07:34.555806 140026528589568 logging_writer.py:48] [85] global_step=85, grad_norm=0.11324, loss=0.128168 +I0914 18:07:34.558829 140069821084864 submission.py:307] 85) loss = 0.128, grad_norm = 0.113 +I0914 18:07:36.672825 140026536982272 logging_writer.py:48] [86] global_step=86, grad_norm=0.202559, loss=0.12725 +I0914 18:07:36.675846 140069821084864 submission.py:307] 86) loss = 0.127, grad_norm = 0.203 +I0914 18:07:39.165729 140026528589568 logging_writer.py:48] [87] global_step=87, grad_norm=0.231787, loss=0.127808 +I0914 18:07:39.168750 140069821084864 submission.py:307] 87) loss = 0.128, grad_norm = 0.232 +I0914 18:07:41.125096 140026536982272 logging_writer.py:48] [88] global_step=88, grad_norm=0.203947, loss=0.127619 +I0914 18:07:41.128239 140069821084864 submission.py:307] 88) loss = 0.128, grad_norm = 0.204 +I0914 18:07:43.362266 140026528589568 logging_writer.py:48] [89] global_step=89, grad_norm=0.135559, loss=0.127602 +I0914 18:07:43.365293 140069821084864 submission.py:307] 89) loss = 0.128, grad_norm = 0.136 +I0914 18:07:45.768435 140026536982272 logging_writer.py:48] [90] global_step=90, grad_norm=0.0912989, loss=0.130664 +I0914 18:07:45.771471 140069821084864 submission.py:307] 90) loss = 0.131, grad_norm = 0.091 +I0914 18:07:47.911296 140026528589568 logging_writer.py:48] [91] global_step=91, grad_norm=0.089513, loss=0.128712 +I0914 18:07:47.914750 140069821084864 submission.py:307] 91) loss = 0.129, grad_norm = 0.090 +I0914 18:07:50.137560 140026536982272 logging_writer.py:48] [92] global_step=92, grad_norm=0.116322, loss=0.129518 +I0914 18:07:50.140608 140069821084864 submission.py:307] 92) loss = 0.130, grad_norm = 0.116 +I0914 18:07:52.433534 140026528589568 logging_writer.py:48] [93] global_step=93, grad_norm=0.235479, loss=0.125931 +I0914 18:07:52.436655 140069821084864 submission.py:307] 93) loss = 0.126, grad_norm = 0.235 +I0914 18:07:54.737554 140026536982272 logging_writer.py:48] [94] global_step=94, grad_norm=0.449339, loss=0.130595 +I0914 18:07:54.740588 140069821084864 submission.py:307] 94) loss = 0.131, grad_norm = 0.449 +I0914 18:07:57.053887 140026528589568 logging_writer.py:48] [95] global_step=95, grad_norm=0.499999, loss=0.129806 +I0914 18:07:57.057636 140069821084864 submission.py:307] 95) loss = 0.130, grad_norm = 0.500 +I0914 18:07:59.154736 140026536982272 logging_writer.py:48] [96] global_step=96, grad_norm=0.163523, loss=0.126275 +I0914 18:07:59.157771 140069821084864 submission.py:307] 96) loss = 0.126, grad_norm = 0.164 +I0914 18:08:01.652904 140026528589568 logging_writer.py:48] [97] global_step=97, grad_norm=0.257663, loss=0.126744 +I0914 18:08:01.655922 140069821084864 submission.py:307] 97) loss = 0.127, grad_norm = 0.258 +I0914 18:08:03.816343 140026536982272 logging_writer.py:48] [98] global_step=98, grad_norm=0.482912, loss=0.127803 +I0914 18:08:03.819383 140069821084864 submission.py:307] 98) loss = 0.128, grad_norm = 0.483 +I0914 18:08:06.121062 140026528589568 logging_writer.py:48] [99] global_step=99, grad_norm=0.395164, loss=0.126927 +I0914 18:08:06.124112 140069821084864 submission.py:307] 99) loss = 0.127, grad_norm = 0.395 +I0914 18:08:08.212876 140026536982272 logging_writer.py:48] [100] global_step=100, grad_norm=0.0259142, loss=0.122931 +I0914 18:08:08.216204 140069821084864 submission.py:307] 100) loss = 0.123, grad_norm = 0.026 +I0914 18:11:16.704610 140069821084864 spec.py:333] Evaluating on the training split. +I0914 18:21:26.855887 140069821084864 spec.py:346] Evaluating on the validation split. +I0914 18:25:04.014981 140069821084864 spec.py:363] Evaluating on the test split. +I0914 18:29:31.283564 140069821084864 submission_runner.py:516] Time since start: 3608.14s, Step: 186, {'train/loss': 0.12694673304587673, 'validation/loss': 0.1290675938772927, 'validation/num_examples': 83274637, 'test/loss': 0.13167683004375258, 'test/num_examples': 95000000, 'score': 367.84259271621704, 'total_duration': 3608.1425683498383, 'accumulated_submission_time': 367.84259271621704, 'accumulated_eval_time': 3239.0816621780396, 'accumulated_logging_time': 0.17610836029052734} +I0914 18:29:31.306059 140026528589568 logging_writer.py:48] [186] accumulated_eval_time=3239.08, accumulated_logging_time=0.176108, accumulated_submission_time=367.843, global_step=186, preemption_count=0, score=367.843, test/loss=0.131677, test/num_examples=95000000, total_duration=3608.14, train/loss=0.126947, validation/loss=0.129068, validation/num_examples=83274637 +I0914 18:35:29.012535 140069821084864 spec.py:333] Evaluating on the training split. +I0914 18:46:26.065652 140069821084864 spec.py:346] Evaluating on the validation split. +I0914 18:50:03.028874 140069821084864 spec.py:363] Evaluating on the test split. +I0914 18:54:26.595230 140069821084864 submission_runner.py:516] Time since start: 5103.45s, Step: 370, {'train/loss': 0.12826613458113487, 'validation/loss': 0.12754104139467642, 'validation/num_examples': 83274637, 'test/loss': 0.130261814970157, 'test/num_examples': 95000000, 'score': 724.8272361755371, 'total_duration': 5103.454279661179, 'accumulated_submission_time': 724.8272361755371, 'accumulated_eval_time': 4376.664420127869, 'accumulated_logging_time': 0.20498251914978027} +I0914 18:54:26.617550 140026536982272 logging_writer.py:48] [370] accumulated_eval_time=4376.66, accumulated_logging_time=0.204983, accumulated_submission_time=724.827, global_step=370, preemption_count=0, score=724.827, test/loss=0.130262, test/num_examples=95000000, total_duration=5103.45, train/loss=0.128266, validation/loss=0.127541, validation/num_examples=83274637 +I0914 18:58:28.114557 140026528589568 logging_writer.py:48] [500] global_step=500, grad_norm=0.0315373, loss=0.130997 +I0914 18:58:28.117753 140069821084864 submission.py:307] 500) loss = 0.131, grad_norm = 0.032 +I0914 19:00:24.639265 140069821084864 spec.py:333] Evaluating on the training split. +I0914 19:10:56.127309 140069821084864 spec.py:346] Evaluating on the validation split. +I0914 19:14:33.734404 140069821084864 spec.py:363] Evaluating on the test split. +I0914 19:18:40.692546 140069821084864 submission_runner.py:516] Time since start: 6557.55s, Step: 553, {'train/loss': 0.1267044816886962, 'validation/loss': 0.12685694044409168, 'validation/num_examples': 83274637, 'test/loss': 0.12932246176918433, 'test/num_examples': 95000000, 'score': 1082.1323177814484, 'total_duration': 6557.551578044891, 'accumulated_submission_time': 1082.1323177814484, 'accumulated_eval_time': 5472.717729330063, 'accumulated_logging_time': 0.23378658294677734} +I0914 19:18:40.714917 140026536982272 logging_writer.py:48] [553] accumulated_eval_time=5472.72, accumulated_logging_time=0.233787, accumulated_submission_time=1082.13, global_step=553, preemption_count=0, score=1082.13, test/loss=0.129322, test/num_examples=95000000, total_duration=6557.55, train/loss=0.126704, validation/loss=0.126857, validation/num_examples=83274637 +I0914 19:24:39.012210 140069821084864 spec.py:333] Evaluating on the training split. +I0914 19:34:56.038390 140069821084864 spec.py:346] Evaluating on the validation split. +I0914 19:38:32.819715 140069821084864 spec.py:363] Evaluating on the test split. +I0914 19:42:40.144806 140069821084864 submission_runner.py:516] Time since start: 7997.00s, Step: 747, {'train/loss': 0.12635178175047054, 'validation/loss': 0.1264811841288897, 'validation/num_examples': 83274637, 'test/loss': 0.12908050685762104, 'test/num_examples': 95000000, 'score': 1439.6988294124603, 'total_duration': 7997.0038385391235, 'accumulated_submission_time': 1439.6988294124603, 'accumulated_eval_time': 6553.85037279129, 'accumulated_logging_time': 0.2626612186431885} +I0914 19:42:40.166199 140026528589568 logging_writer.py:48] [747] accumulated_eval_time=6553.85, accumulated_logging_time=0.262661, accumulated_submission_time=1439.7, global_step=747, preemption_count=0, score=1439.7, test/loss=0.129081, test/num_examples=95000000, total_duration=7997, train/loss=0.126352, validation/loss=0.126481, validation/num_examples=83274637 +I0914 19:48:36.906913 140069821084864 spec.py:333] Evaluating on the training split. +I0914 19:59:33.096648 140069821084864 spec.py:346] Evaluating on the validation split. +I0914 20:03:10.612035 140069821084864 spec.py:363] Evaluating on the test split. +I0914 20:07:17.339934 140069821084864 submission_runner.py:516] Time since start: 9474.20s, Step: 956, {'train/loss': 0.12435807907188352, 'validation/loss': 0.12621115180701778, 'validation/num_examples': 83274637, 'test/loss': 0.12875099081501207, 'test/num_examples': 95000000, 'score': 1795.705944776535, 'total_duration': 9474.198992490768, 'accumulated_submission_time': 1795.705944776535, 'accumulated_eval_time': 7674.283470630646, 'accumulated_logging_time': 0.2904345989227295} +I0914 20:07:17.375529 140026536982272 logging_writer.py:48] [956] accumulated_eval_time=7674.28, accumulated_logging_time=0.290435, accumulated_submission_time=1795.71, global_step=956, preemption_count=0, score=1795.71, test/loss=0.128751, test/num_examples=95000000, total_duration=9474.2, train/loss=0.124358, validation/loss=0.126211, validation/num_examples=83274637 +I0914 20:08:00.499643 140026528589568 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.0400113, loss=0.123219 +I0914 20:08:00.503364 140069821084864 submission.py:307] 1000) loss = 0.123, grad_norm = 0.040 +I0914 20:13:14.744668 140069821084864 spec.py:333] Evaluating on the training split. +I0914 20:23:31.304592 140069821084864 spec.py:346] Evaluating on the validation split. +I0914 20:27:07.881846 140069821084864 spec.py:363] Evaluating on the test split. +I0914 20:31:14.055181 140069821084864 submission_runner.py:516] Time since start: 10910.91s, Step: 1171, {'train/loss': 0.1261888873957049, 'validation/loss': 0.12637783631326305, 'validation/num_examples': 83274637, 'test/loss': 0.12909958038892244, 'test/num_examples': 95000000, 'score': 2152.3411235809326, 'total_duration': 10910.914236068726, 'accumulated_submission_time': 2152.3411235809326, 'accumulated_eval_time': 8753.594065904617, 'accumulated_logging_time': 0.33248448371887207} +I0914 20:31:14.077235 140026536982272 logging_writer.py:48] [1171] accumulated_eval_time=8753.59, accumulated_logging_time=0.332484, accumulated_submission_time=2152.34, global_step=1171, preemption_count=0, score=2152.34, test/loss=0.1291, test/num_examples=95000000, total_duration=10910.9, train/loss=0.126189, validation/loss=0.126378, validation/num_examples=83274637 +I0914 20:37:10.662388 140069821084864 spec.py:333] Evaluating on the training split. +I0914 20:46:03.378149 140069821084864 spec.py:346] Evaluating on the validation split. +I0914 20:49:38.078608 140069821084864 spec.py:363] Evaluating on the test split. +I0914 20:53:42.679903 140069821084864 submission_runner.py:516] Time since start: 12259.54s, Step: 1358, {'train/loss': 0.12328442864427089, 'validation/loss': 0.12611475707139086, 'validation/num_examples': 83274637, 'test/loss': 0.1283687012205425, 'test/num_examples': 95000000, 'score': 2508.20432639122, 'total_duration': 12259.538949489594, 'accumulated_submission_time': 2508.20432639122, 'accumulated_eval_time': 9745.611643075943, 'accumulated_logging_time': 0.36100244522094727} +I0914 20:53:42.701343 140026528589568 logging_writer.py:48] [1358] accumulated_eval_time=9745.61, accumulated_logging_time=0.361002, accumulated_submission_time=2508.2, global_step=1358, preemption_count=0, score=2508.2, test/loss=0.128369, test/num_examples=95000000, total_duration=12259.5, train/loss=0.123284, validation/loss=0.126115, validation/num_examples=83274637 +I0914 20:57:57.251245 140026536982272 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.0107403, loss=0.129935 +I0914 20:57:57.254611 140069821084864 submission.py:307] 1500) loss = 0.130, grad_norm = 0.011 +I0914 20:59:39.378459 140069821084864 spec.py:333] Evaluating on the training split. +I0914 21:09:23.705406 140069821084864 spec.py:346] Evaluating on the validation split. +I0914 21:12:57.005846 140069821084864 spec.py:363] Evaluating on the test split. +I0914 21:16:59.790764 140069821084864 submission_runner.py:516] Time since start: 13656.65s, Step: 1549, {'train/loss': 0.12528120088089603, 'validation/loss': 0.12568715767984323, 'validation/num_examples': 83274637, 'test/loss': 0.12794226515808105, 'test/num_examples': 95000000, 'score': 2864.1562666893005, 'total_duration': 13656.649791002274, 'accumulated_submission_time': 2864.1562666893005, 'accumulated_eval_time': 10786.024005651474, 'accumulated_logging_time': 0.38872313499450684} +I0914 21:16:59.813406 140026528589568 logging_writer.py:48] [1549] accumulated_eval_time=10786, accumulated_logging_time=0.388723, accumulated_submission_time=2864.16, global_step=1549, preemption_count=0, score=2864.16, test/loss=0.127942, test/num_examples=95000000, total_duration=13656.6, train/loss=0.125281, validation/loss=0.125687, validation/num_examples=83274637 +I0914 21:22:56.407713 140069821084864 spec.py:333] Evaluating on the training split. +I0914 21:32:51.206402 140069821084864 spec.py:346] Evaluating on the validation split. +I0914 21:36:20.097892 140069821084864 spec.py:363] Evaluating on the test split. +I0914 21:40:17.691931 140069821084864 submission_runner.py:516] Time since start: 15054.55s, Step: 1748, {'train/loss': 0.12534069006365894, 'validation/loss': 0.1254944849850555, 'validation/num_examples': 83274637, 'test/loss': 0.12791959633997868, 'test/num_examples': 95000000, 'score': 3220.04137635231, 'total_duration': 15054.550971746445, 'accumulated_submission_time': 3220.04137635231, 'accumulated_eval_time': 11827.308275699615, 'accumulated_logging_time': 0.41773366928100586} +I0914 21:40:17.713609 140026536982272 logging_writer.py:48] [1748] accumulated_eval_time=11827.3, accumulated_logging_time=0.417734, accumulated_submission_time=3220.04, global_step=1748, preemption_count=0, score=3220.04, test/loss=0.12792, test/num_examples=95000000, total_duration=15054.6, train/loss=0.125341, validation/loss=0.125494, validation/num_examples=83274637 +I0914 21:46:14.608562 140069821084864 spec.py:333] Evaluating on the training split. +I0914 21:54:51.357386 140069821084864 spec.py:346] Evaluating on the validation split. +I0914 21:58:10.791788 140069821084864 spec.py:363] Evaluating on the test split. +I0914 22:01:58.225624 140069821084864 submission_runner.py:516] Time since start: 16355.08s, Step: 1953, {'train/loss': 0.12294707671647, 'validation/loss': 0.1253609903469546, 'validation/num_examples': 83274637, 'test/loss': 0.12768890750491493, 'test/num_examples': 95000000, 'score': 3576.227335691452, 'total_duration': 16355.084664106369, 'accumulated_submission_time': 3576.227335691452, 'accumulated_eval_time': 12770.925372123718, 'accumulated_logging_time': 0.44575023651123047} +I0914 22:01:58.249147 140026528589568 logging_writer.py:48] [1953] accumulated_eval_time=12770.9, accumulated_logging_time=0.44575, accumulated_submission_time=3576.23, global_step=1953, preemption_count=0, score=3576.23, test/loss=0.127689, test/num_examples=95000000, total_duration=16355.1, train/loss=0.122947, validation/loss=0.125361, validation/num_examples=83274637 +I0914 22:02:39.570953 140026536982272 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.00970958, loss=0.125704 +I0914 22:02:39.574080 140069821084864 submission.py:307] 2000) loss = 0.126, grad_norm = 0.010 +I0914 22:07:55.280849 140069821084864 spec.py:333] Evaluating on the training split. +I0914 22:14:46.845383 140069821084864 spec.py:346] Evaluating on the validation split. +I0914 22:17:45.261638 140069821084864 spec.py:363] Evaluating on the test split. +I0914 22:21:11.879209 140069821084864 submission_runner.py:516] Time since start: 17508.74s, Step: 2167, {'train/loss': 0.1231787101783924, 'validation/loss': 0.12556251450551098, 'validation/num_examples': 83274637, 'test/loss': 0.12800871344732986, 'test/num_examples': 95000000, 'score': 3932.574423313141, 'total_duration': 17508.73826622963, 'accumulated_submission_time': 3932.574423313141, 'accumulated_eval_time': 13567.52386879921, 'accumulated_logging_time': 0.47588253021240234} +I0914 22:21:11.901387 140026528589568 logging_writer.py:48] [2167] accumulated_eval_time=13567.5, accumulated_logging_time=0.475883, accumulated_submission_time=3932.57, global_step=2167, preemption_count=0, score=3932.57, test/loss=0.128009, test/num_examples=95000000, total_duration=17508.7, train/loss=0.123179, validation/loss=0.125563, validation/num_examples=83274637 +I0914 22:27:08.986437 140069821084864 spec.py:333] Evaluating on the training split. +I0914 22:32:12.504598 140069821084864 spec.py:346] Evaluating on the validation split. +I0914 22:35:10.896489 140069821084864 spec.py:363] Evaluating on the test split. +I0914 22:38:42.884100 140069821084864 submission_runner.py:516] Time since start: 18559.74s, Step: 2369, {'train/loss': 0.12290766619101164, 'validation/loss': 0.12530956720384262, 'validation/num_examples': 83274637, 'test/loss': 0.12779830862041272, 'test/num_examples': 95000000, 'score': 4288.982084035873, 'total_duration': 18559.743125915527, 'accumulated_submission_time': 4288.982084035873, 'accumulated_eval_time': 14261.421571493149, 'accumulated_logging_time': 0.504258394241333} +I0914 22:38:42.906040 140026536982272 logging_writer.py:48] [2369] accumulated_eval_time=14261.4, accumulated_logging_time=0.504258, accumulated_submission_time=4288.98, global_step=2369, preemption_count=0, score=4288.98, test/loss=0.127798, test/num_examples=95000000, total_duration=18559.7, train/loss=0.122908, validation/loss=0.12531, validation/num_examples=83274637 +I0914 22:42:08.194474 140026528589568 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.0150674, loss=0.124488 +I0914 22:42:08.197782 140069821084864 submission.py:307] 2500) loss = 0.124, grad_norm = 0.015 +I0914 22:44:41.068930 140069821084864 spec.py:333] Evaluating on the training split. +I0914 22:48:26.872511 140069821084864 spec.py:346] Evaluating on the validation split. +I0914 22:51:25.003875 140069821084864 spec.py:363] Evaluating on the test split. +I0914 22:54:57.440474 140069821084864 submission_runner.py:516] Time since start: 19534.30s, Step: 2578, {'train/loss': 0.12351121791821107, 'validation/loss': 0.12480875427967063, 'validation/num_examples': 83274637, 'test/loss': 0.1271974353258635, 'test/num_examples': 95000000, 'score': 4646.4661474227905, 'total_duration': 19534.2995262146, 'accumulated_submission_time': 4646.4661474227905, 'accumulated_eval_time': 14877.79320716858, 'accumulated_logging_time': 0.5324914455413818} +I0914 22:54:57.463219 140026536982272 logging_writer.py:48] [2578] accumulated_eval_time=14877.8, accumulated_logging_time=0.532491, accumulated_submission_time=4646.47, global_step=2578, preemption_count=0, score=4646.47, test/loss=0.127197, test/num_examples=95000000, total_duration=19534.3, train/loss=0.123511, validation/loss=0.124809, validation/num_examples=83274637 +I0914 23:00:55.268599 140069821084864 spec.py:333] Evaluating on the training split. +I0914 23:02:35.552597 140069821084864 spec.py:346] Evaluating on the validation split. +I0914 23:05:34.120453 140069821084864 spec.py:363] Evaluating on the test split. +I0914 23:09:07.235566 140069821084864 submission_runner.py:516] Time since start: 20384.09s, Step: 2781, {'train/loss': 0.12444366294393741, 'validation/loss': 0.12473375667010844, 'validation/num_examples': 83274637, 'test/loss': 0.12709752272370992, 'test/num_examples': 95000000, 'score': 5003.59354186058, 'total_duration': 20384.09462237358, 'accumulated_submission_time': 5003.59354186058, 'accumulated_eval_time': 15369.760269403458, 'accumulated_logging_time': 0.5615274906158447} +I0914 23:09:07.257657 140026528589568 logging_writer.py:48] [2781] accumulated_eval_time=15369.8, accumulated_logging_time=0.561527, accumulated_submission_time=5003.59, global_step=2781, preemption_count=0, score=5003.59, test/loss=0.127098, test/num_examples=95000000, total_duration=20384.1, train/loss=0.124444, validation/loss=0.124734, validation/num_examples=83274637 +I0914 23:15:04.548539 140069821084864 spec.py:333] Evaluating on the training split. +I0914 23:16:02.874647 140069821084864 spec.py:346] Evaluating on the validation split. +I0914 23:19:01.247142 140069821084864 spec.py:363] Evaluating on the test split. +I0914 23:22:33.609757 140069821084864 submission_runner.py:516] Time since start: 21190.47s, Step: 2976, {'train/loss': 0.12191889421237856, 'validation/loss': 0.12478362670817317, 'validation/num_examples': 83274637, 'test/loss': 0.1271950413490697, 'test/num_examples': 95000000, 'score': 5360.208089351654, 'total_duration': 21190.468819379807, 'accumulated_submission_time': 5360.208089351654, 'accumulated_eval_time': 15818.821569442749, 'accumulated_logging_time': 0.5901992321014404} +I0914 23:22:33.630639 140026536982272 logging_writer.py:48] [2976] accumulated_eval_time=15818.8, accumulated_logging_time=0.590199, accumulated_submission_time=5360.21, global_step=2976, preemption_count=0, score=5360.21, test/loss=0.127195, test/num_examples=95000000, total_duration=21190.5, train/loss=0.121919, validation/loss=0.124784, validation/num_examples=83274637 +I0914 23:22:39.558706 140026528589568 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.00681748, loss=0.122913 +I0914 23:22:39.561671 140069821084864 submission.py:307] 3000) loss = 0.123, grad_norm = 0.007 +I0914 23:28:30.010405 140069821084864 spec.py:333] Evaluating on the training split. +I0914 23:29:28.302504 140069821084864 spec.py:346] Evaluating on the validation split. +I0914 23:32:26.414805 140069821084864 spec.py:363] Evaluating on the test split. +I0914 23:35:58.671257 140069821084864 submission_runner.py:516] Time since start: 21995.53s, Step: 3204, {'train/loss': 0.12320158136273823, 'validation/loss': 0.12468983266495527, 'validation/num_examples': 83274637, 'test/loss': 0.1270577581971821, 'test/num_examples': 95000000, 'score': 5715.89444732666, 'total_duration': 21995.530319452286, 'accumulated_submission_time': 5715.89444732666, 'accumulated_eval_time': 16267.482496500015, 'accumulated_logging_time': 0.6184685230255127} +I0914 23:35:58.693293 140026536982272 logging_writer.py:48] [3204] accumulated_eval_time=16267.5, accumulated_logging_time=0.618469, accumulated_submission_time=5715.89, global_step=3204, preemption_count=0, score=5715.89, test/loss=0.127058, test/num_examples=95000000, total_duration=21995.5, train/loss=0.123202, validation/loss=0.12469, validation/num_examples=83274637 +I0914 23:41:57.048506 140069821084864 spec.py:333] Evaluating on the training split. +I0914 23:42:55.592089 140069821084864 spec.py:346] Evaluating on the validation split. +I0914 23:45:53.913392 140069821084864 spec.py:363] Evaluating on the test split. +I0914 23:49:28.110052 140069821084864 submission_runner.py:516] Time since start: 22804.97s, Step: 3402, {'train/loss': 0.12404982798543915, 'validation/loss': 0.12470717372903027, 'validation/num_examples': 83274637, 'test/loss': 0.12714490267510664, 'test/num_examples': 95000000, 'score': 6073.568929672241, 'total_duration': 22804.969116687775, 'accumulated_submission_time': 6073.568929672241, 'accumulated_eval_time': 16718.54413294792, 'accumulated_logging_time': 0.6468093395233154} +I0914 23:49:28.132344 140026528589568 logging_writer.py:48] [3402] accumulated_eval_time=16718.5, accumulated_logging_time=0.646809, accumulated_submission_time=6073.57, global_step=3402, preemption_count=0, score=6073.57, test/loss=0.127145, test/num_examples=95000000, total_duration=22805, train/loss=0.12405, validation/loss=0.124707, validation/num_examples=83274637 +I0914 23:51:58.047083 140026536982272 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.00721512, loss=0.121678 +I0914 23:51:58.050323 140069821084864 submission.py:307] 3500) loss = 0.122, grad_norm = 0.007 +I0914 23:55:25.218576 140069821084864 spec.py:333] Evaluating on the training split. +I0914 23:56:23.866321 140069821084864 spec.py:346] Evaluating on the validation split. +I0914 23:59:22.187195 140069821084864 spec.py:363] Evaluating on the test split. +I0915 00:02:57.050585 140069821084864 submission_runner.py:516] Time since start: 23613.91s, Step: 3612, {'train/loss': 0.12195268458024051, 'validation/loss': 0.1246549948902484, 'validation/num_examples': 83274637, 'test/loss': 0.1271164032991108, 'test/num_examples': 95000000, 'score': 6429.9725069999695, 'total_duration': 23613.909635543823, 'accumulated_submission_time': 6429.9725069999695, 'accumulated_eval_time': 17170.37621164322, 'accumulated_logging_time': 0.6755161285400391} +I0915 00:02:57.073406 140026528589568 logging_writer.py:48] [3612] accumulated_eval_time=17170.4, accumulated_logging_time=0.675516, accumulated_submission_time=6429.97, global_step=3612, preemption_count=0, score=6429.97, test/loss=0.127116, test/num_examples=95000000, total_duration=23613.9, train/loss=0.121953, validation/loss=0.124655, validation/num_examples=83274637 +I0915 00:08:54.415119 140069821084864 spec.py:333] Evaluating on the training split. +I0915 00:09:52.994674 140069821084864 spec.py:346] Evaluating on the validation split. +I0915 00:12:51.488348 140069821084864 spec.py:363] Evaluating on the test split. +I0915 00:16:24.279486 140069821084864 submission_runner.py:516] Time since start: 24421.14s, Step: 3813, {'train/loss': 0.12360765451201504, 'validation/loss': 0.12468748179670476, 'validation/num_examples': 83274637, 'test/loss': 0.12715450920169227, 'test/num_examples': 95000000, 'score': 6786.63451218605, 'total_duration': 24421.138518333435, 'accumulated_submission_time': 6786.63451218605, 'accumulated_eval_time': 17620.24062728882, 'accumulated_logging_time': 0.7046818733215332} +I0915 00:16:24.319463 140026536982272 logging_writer.py:48] [3813] accumulated_eval_time=17620.2, accumulated_logging_time=0.704682, accumulated_submission_time=6786.63, global_step=3813, preemption_count=0, score=6786.63, test/loss=0.127155, test/num_examples=95000000, total_duration=24421.1, train/loss=0.123608, validation/loss=0.124687, validation/num_examples=83274637 +I0915 00:22:02.036281 140026528589568 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.0127975, loss=0.120338 +I0915 00:22:02.039498 140069821084864 submission.py:307] 4000) loss = 0.120, grad_norm = 0.013 +I0915 00:22:21.670411 140069821084864 spec.py:333] Evaluating on the training split. +I0915 00:23:20.208191 140069821084864 spec.py:346] Evaluating on the validation split. +I0915 00:26:19.089821 140069821084864 spec.py:363] Evaluating on the test split. +I0915 00:29:53.292717 140069821084864 submission_runner.py:516] Time since start: 25230.15s, Step: 4010, {'train/loss': 0.12330253023457075, 'validation/loss': 0.12468694847849297, 'validation/num_examples': 83274637, 'test/loss': 0.12709274321465744, 'test/num_examples': 95000000, 'score': 7143.305164813995, 'total_duration': 25230.151753425598, 'accumulated_submission_time': 7143.305164813995, 'accumulated_eval_time': 18071.862988710403, 'accumulated_logging_time': 0.7509946823120117} +I0915 00:29:53.315381 140026536982272 logging_writer.py:48] [4010] accumulated_eval_time=18071.9, accumulated_logging_time=0.750995, accumulated_submission_time=7143.31, global_step=4010, preemption_count=0, score=7143.31, test/loss=0.127093, test/num_examples=95000000, total_duration=25230.2, train/loss=0.123303, validation/loss=0.124687, validation/num_examples=83274637 +I0915 00:35:50.337291 140069821084864 spec.py:333] Evaluating on the training split. +I0915 00:36:48.631671 140069821084864 spec.py:346] Evaluating on the validation split. +I0915 00:39:47.088165 140069821084864 spec.py:363] Evaluating on the test split. +I0915 00:43:21.432877 140069821084864 submission_runner.py:516] Time since start: 26038.29s, Step: 4238, {'train/loss': 0.12299114936688256, 'validation/loss': 0.12447192473189195, 'validation/num_examples': 83274637, 'test/loss': 0.12683050084485506, 'test/num_examples': 95000000, 'score': 7499.6366539001465, 'total_duration': 26038.29193878174, 'accumulated_submission_time': 7499.6366539001465, 'accumulated_eval_time': 18522.95866727829, 'accumulated_logging_time': 0.7810301780700684} +I0915 00:43:21.454982 140026528589568 logging_writer.py:48] [4238] accumulated_eval_time=18523, accumulated_logging_time=0.78103, accumulated_submission_time=7499.64, global_step=4238, preemption_count=0, score=7499.64, test/loss=0.126831, test/num_examples=95000000, total_duration=26038.3, train/loss=0.122991, validation/loss=0.124472, validation/num_examples=83274637 +I0915 00:49:18.116367 140069821084864 spec.py:333] Evaluating on the training split. +I0915 00:50:16.565878 140069821084864 spec.py:346] Evaluating on the validation split. +I0915 00:53:15.019662 140069821084864 spec.py:363] Evaluating on the test split. +I0915 00:56:51.393243 140069821084864 submission_runner.py:516] Time since start: 26848.25s, Step: 4450, {'train/loss': 0.12318694136432898, 'validation/loss': 0.12442490228230692, 'validation/num_examples': 83274637, 'test/loss': 0.1267312048434609, 'test/num_examples': 95000000, 'score': 7855.612186908722, 'total_duration': 26848.252278089523, 'accumulated_submission_time': 7855.612186908722, 'accumulated_eval_time': 18976.235602855682, 'accumulated_logging_time': 0.8097052574157715} +I0915 00:56:51.416056 140026536982272 logging_writer.py:48] [4450] accumulated_eval_time=18976.2, accumulated_logging_time=0.809705, accumulated_submission_time=7855.61, global_step=4450, preemption_count=0, score=7855.61, test/loss=0.126731, test/num_examples=95000000, total_duration=26848.3, train/loss=0.123187, validation/loss=0.124425, validation/num_examples=83274637 +I0915 00:57:42.765811 140026528589568 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.00550038, loss=0.122811 +I0915 00:57:42.768931 140069821084864 submission.py:307] 4500) loss = 0.123, grad_norm = 0.006 +I0915 01:02:48.888706 140069821084864 spec.py:333] Evaluating on the training split. +I0915 01:03:47.393203 140069821084864 spec.py:346] Evaluating on the validation split. +I0915 01:06:45.761027 140069821084864 spec.py:363] Evaluating on the test split. +I0915 01:10:19.758523 140069821084864 submission_runner.py:516] Time since start: 27656.62s, Step: 4655, {'train/loss': 0.12421156305302328, 'validation/loss': 0.12446358872719832, 'validation/num_examples': 83274637, 'test/loss': 0.12681963255470677, 'test/num_examples': 95000000, 'score': 8212.401973485947, 'total_duration': 27656.617559194565, 'accumulated_submission_time': 8212.401973485947, 'accumulated_eval_time': 19427.105493545532, 'accumulated_logging_time': 0.8389811515808105} +I0915 01:10:19.781255 140026536982272 logging_writer.py:48] [4655] accumulated_eval_time=19427.1, accumulated_logging_time=0.838981, accumulated_submission_time=8212.4, global_step=4655, preemption_count=0, score=8212.4, test/loss=0.12682, test/num_examples=95000000, total_duration=27656.6, train/loss=0.124212, validation/loss=0.124464, validation/num_examples=83274637 +I0915 01:16:17.245404 140069821084864 spec.py:333] Evaluating on the training split. +I0915 01:17:15.725483 140069821084864 spec.py:346] Evaluating on the validation split. +I0915 01:20:14.462506 140069821084864 spec.py:363] Evaluating on the test split. +I0915 01:23:48.316010 140069821084864 submission_runner.py:516] Time since start: 28465.18s, Step: 4872, {'train/loss': 0.12338476969904284, 'validation/loss': 0.12444184406496256, 'validation/num_examples': 83274637, 'test/loss': 0.1266996562864605, 'test/num_examples': 95000000, 'score': 8569.17642068863, 'total_duration': 28465.175060749054, 'accumulated_submission_time': 8569.17642068863, 'accumulated_eval_time': 19878.17617082596, 'accumulated_logging_time': 0.868128776550293} +I0915 01:23:48.338837 140026528589568 logging_writer.py:48] [4872] accumulated_eval_time=19878.2, accumulated_logging_time=0.868129, accumulated_submission_time=8569.18, global_step=4872, preemption_count=0, score=8569.18, test/loss=0.1267, test/num_examples=95000000, total_duration=28465.2, train/loss=0.123385, validation/loss=0.124442, validation/num_examples=83274637 +I0915 01:27:08.328621 140026536982272 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.0194436, loss=0.120582 +I0915 01:27:08.331904 140069821084864 submission.py:307] 5000) loss = 0.121, grad_norm = 0.019 +I0915 01:29:45.169800 140069821084864 spec.py:333] Evaluating on the training split. +I0915 01:30:43.622518 140069821084864 spec.py:346] Evaluating on the validation split. +I0915 01:33:42.109987 140069821084864 spec.py:363] Evaluating on the test split. +I0915 01:37:17.351616 140069821084864 submission_runner.py:516] Time since start: 29274.21s, Step: 5083, {'train/loss': 0.12048792654416463, 'validation/loss': 0.12430243447180923, 'validation/num_examples': 83274637, 'test/loss': 0.12654046956674675, 'test/num_examples': 95000000, 'score': 8925.319944858551, 'total_duration': 29274.210667848587, 'accumulated_submission_time': 8925.319944858551, 'accumulated_eval_time': 20330.358067274094, 'accumulated_logging_time': 0.8973524570465088} +I0915 01:37:17.375989 140026528589568 logging_writer.py:48] [5083] accumulated_eval_time=20330.4, accumulated_logging_time=0.897352, accumulated_submission_time=8925.32, global_step=5083, preemption_count=0, score=8925.32, test/loss=0.12654, test/num_examples=95000000, total_duration=29274.2, train/loss=0.120488, validation/loss=0.124302, validation/num_examples=83274637 +I0915 01:43:15.042099 140069821084864 spec.py:333] Evaluating on the training split. +I0915 01:44:13.439768 140069821084864 spec.py:346] Evaluating on the validation split. +I0915 01:47:11.959856 140069821084864 spec.py:363] Evaluating on the test split. +I0915 01:50:45.994022 140069821084864 submission_runner.py:516] Time since start: 30082.85s, Step: 5295, {'train/loss': 0.12289239757459962, 'validation/loss': 0.12426028797097123, 'validation/num_examples': 83274637, 'test/loss': 0.12665145451579846, 'test/num_examples': 95000000, 'score': 9282.303190946579, 'total_duration': 30082.853084802628, 'accumulated_submission_time': 9282.303190946579, 'accumulated_eval_time': 20781.310081481934, 'accumulated_logging_time': 0.9282863140106201} +I0915 01:50:46.018854 140026536982272 logging_writer.py:48] [5295] accumulated_eval_time=20781.3, accumulated_logging_time=0.928286, accumulated_submission_time=9282.3, global_step=5295, preemption_count=0, score=9282.3, test/loss=0.126651, test/num_examples=95000000, total_duration=30082.9, train/loss=0.122892, validation/loss=0.12426, validation/num_examples=83274637 +I0915 01:56:26.433485 140026528589568 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.0061199, loss=0.125416 +I0915 01:56:26.436708 140069821084864 submission.py:307] 5500) loss = 0.125, grad_norm = 0.006 +I0915 01:56:44.092236 140069821084864 spec.py:333] Evaluating on the training split. +I0915 01:57:42.505496 140069821084864 spec.py:346] Evaluating on the validation split. +I0915 02:00:41.519892 140069821084864 spec.py:363] Evaluating on the test split. +I0915 02:04:16.436368 140069821084864 submission_runner.py:516] Time since start: 30893.30s, Step: 5510, {'train/loss': 0.12264581239439097, 'validation/loss': 0.12424266595137412, 'validation/num_examples': 83274637, 'test/loss': 0.1266076050482499, 'test/num_examples': 95000000, 'score': 9639.686212778091, 'total_duration': 30893.29542040825, 'accumulated_submission_time': 9639.686212778091, 'accumulated_eval_time': 21233.654284477234, 'accumulated_logging_time': 0.9595811367034912} +I0915 02:04:16.474522 140026536982272 logging_writer.py:48] [5510] accumulated_eval_time=21233.7, accumulated_logging_time=0.959581, accumulated_submission_time=9639.69, global_step=5510, preemption_count=0, score=9639.69, test/loss=0.126608, test/num_examples=95000000, total_duration=30893.3, train/loss=0.122646, validation/loss=0.124243, validation/num_examples=83274637 +I0915 02:10:14.312966 140069821084864 spec.py:333] Evaluating on the training split. +I0915 02:11:12.896099 140069821084864 spec.py:346] Evaluating on the validation split. +I0915 02:14:11.763353 140069821084864 spec.py:363] Evaluating on the test split. +I0915 02:17:46.310327 140069821084864 submission_runner.py:516] Time since start: 31703.17s, Step: 5735, {'train/loss': 0.12054924197537491, 'validation/loss': 0.1241994243948032, 'validation/num_examples': 83274637, 'test/loss': 0.12658224439327842, 'test/num_examples': 95000000, 'score': 9996.833044290543, 'total_duration': 31703.16938185692, 'accumulated_submission_time': 9996.833044290543, 'accumulated_eval_time': 21685.651729106903, 'accumulated_logging_time': 1.0043079853057861} +I0915 02:17:46.333949 140026528589568 logging_writer.py:48] [5735] accumulated_eval_time=21685.7, accumulated_logging_time=1.00431, accumulated_submission_time=9996.83, global_step=5735, preemption_count=0, score=9996.83, test/loss=0.126582, test/num_examples=95000000, total_duration=31703.2, train/loss=0.120549, validation/loss=0.124199, validation/num_examples=83274637 +I0915 02:23:43.246392 140069821084864 spec.py:333] Evaluating on the training split. +I0915 02:24:41.724259 140069821084864 spec.py:346] Evaluating on the validation split. +I0915 02:27:40.490787 140069821084864 spec.py:363] Evaluating on the test split. +I0915 02:31:15.188155 140069821084864 submission_runner.py:516] Time since start: 32512.05s, Step: 5955, {'train/loss': 0.12028691511708944, 'validation/loss': 0.12395168462638755, 'validation/num_examples': 83274637, 'test/loss': 0.12629723196443257, 'test/num_examples': 95000000, 'score': 10353.056209802628, 'total_duration': 32512.04721045494, 'accumulated_submission_time': 10353.056209802628, 'accumulated_eval_time': 22137.593573093414, 'accumulated_logging_time': 1.0345714092254639} +I0915 02:31:15.211662 140026536982272 logging_writer.py:48] [5955] accumulated_eval_time=22137.6, accumulated_logging_time=1.03457, accumulated_submission_time=10353.1, global_step=5955, preemption_count=0, score=10353.1, test/loss=0.126297, test/num_examples=95000000, total_duration=32512, train/loss=0.120287, validation/loss=0.123952, validation/num_examples=83274637 +I0915 02:31:56.557355 140026528589568 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.0189949, loss=0.12202 +I0915 02:31:56.560295 140069821084864 submission.py:307] 6000) loss = 0.122, grad_norm = 0.019 +I0915 02:37:11.615815 140069821084864 spec.py:333] Evaluating on the training split. +I0915 02:38:09.902260 140069821084864 spec.py:346] Evaluating on the validation split. +I0915 02:41:09.126507 140069821084864 spec.py:363] Evaluating on the test split. +I0915 02:44:43.683113 140069821084864 submission_runner.py:516] Time since start: 33320.54s, Step: 6175, {'train/loss': 0.12217553914858997, 'validation/loss': 0.12412426897413466, 'validation/num_examples': 83274637, 'test/loss': 0.12650605329573783, 'test/num_examples': 95000000, 'score': 10708.771770477295, 'total_duration': 33320.54216980934, 'accumulated_submission_time': 10708.771770477295, 'accumulated_eval_time': 22589.660952091217, 'accumulated_logging_time': 1.0645341873168945} +I0915 02:44:43.706578 140026536982272 logging_writer.py:48] [6175] accumulated_eval_time=22589.7, accumulated_logging_time=1.06453, accumulated_submission_time=10708.8, global_step=6175, preemption_count=0, score=10708.8, test/loss=0.126506, test/num_examples=95000000, total_duration=33320.5, train/loss=0.122176, validation/loss=0.124124, validation/num_examples=83274637 +I0915 02:50:40.644901 140069821084864 spec.py:333] Evaluating on the training split. +I0915 02:51:39.162285 140069821084864 spec.py:346] Evaluating on the validation split. +I0915 02:54:38.148802 140069821084864 spec.py:363] Evaluating on the test split. +I0915 02:58:10.935821 140069821084864 submission_runner.py:516] Time since start: 34127.79s, Step: 6388, {'train/loss': 0.12235956073112195, 'validation/loss': 0.12391907428857017, 'validation/num_examples': 83274637, 'test/loss': 0.12621670044459293, 'test/num_examples': 95000000, 'score': 11065.027774333954, 'total_duration': 34127.79482841492, 'accumulated_submission_time': 11065.027774333954, 'accumulated_eval_time': 23039.951890945435, 'accumulated_logging_time': 1.0944933891296387} +I0915 02:58:10.960795 140026528589568 logging_writer.py:48] [6388] accumulated_eval_time=23040, accumulated_logging_time=1.09449, accumulated_submission_time=11065, global_step=6388, preemption_count=0, score=11065, test/loss=0.126217, test/num_examples=95000000, total_duration=34127.8, train/loss=0.12236, validation/loss=0.123919, validation/num_examples=83274637 +I0915 03:00:39.785634 140026536982272 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.0145496, loss=0.119019 +I0915 03:00:39.788640 140069821084864 submission.py:307] 6500) loss = 0.119, grad_norm = 0.015 +I0915 03:04:07.798730 140069821084864 spec.py:333] Evaluating on the training split. +I0915 03:05:06.275288 140069821084864 spec.py:346] Evaluating on the validation split. +I0915 03:08:05.027626 140069821084864 spec.py:363] Evaluating on the test split. +I0915 03:11:38.605827 140069821084864 submission_runner.py:516] Time since start: 34935.46s, Step: 6628, {'train/loss': 0.12171748654758212, 'validation/loss': 0.1238195275884534, 'validation/num_examples': 83274637, 'test/loss': 0.12615040450391268, 'test/num_examples': 95000000, 'score': 11421.166724920273, 'total_duration': 34935.46488428116, 'accumulated_submission_time': 11421.166724920273, 'accumulated_eval_time': 23490.759078502655, 'accumulated_logging_time': 1.1259403228759766} +I0915 03:11:38.628466 140026528589568 logging_writer.py:48] [6628] accumulated_eval_time=23490.8, accumulated_logging_time=1.12594, accumulated_submission_time=11421.2, global_step=6628, preemption_count=0, score=11421.2, test/loss=0.12615, test/num_examples=95000000, total_duration=34935.5, train/loss=0.121717, validation/loss=0.12382, validation/num_examples=83274637 +I0915 03:17:36.621032 140069821084864 spec.py:333] Evaluating on the training split. +I0915 03:18:35.191182 140069821084864 spec.py:346] Evaluating on the validation split. +I0915 03:21:34.004299 140069821084864 spec.py:363] Evaluating on the test split. +I0915 03:25:07.693807 140069821084864 submission_runner.py:516] Time since start: 35744.55s, Step: 6864, {'train/loss': 0.12146207886837246, 'validation/loss': 0.12377636809639898, 'validation/num_examples': 83274637, 'test/loss': 0.12609451138241418, 'test/num_examples': 95000000, 'score': 11778.461151599884, 'total_duration': 35744.55286478996, 'accumulated_submission_time': 11778.461151599884, 'accumulated_eval_time': 23941.831956148148, 'accumulated_logging_time': 1.1551685333251953} +I0915 03:25:07.717705 140026536982272 logging_writer.py:48] [6864] accumulated_eval_time=23941.8, accumulated_logging_time=1.15517, accumulated_submission_time=11778.5, global_step=6864, preemption_count=0, score=11778.5, test/loss=0.126095, test/num_examples=95000000, total_duration=35744.6, train/loss=0.121462, validation/loss=0.123776, validation/num_examples=83274637 +I0915 03:28:28.437519 140026528589568 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.00727773, loss=0.123088 +I0915 03:28:28.440638 140069821084864 submission.py:307] 7000) loss = 0.123, grad_norm = 0.007 +I0915 03:31:04.676876 140069821084864 spec.py:333] Evaluating on the training split. +I0915 03:32:03.161225 140069821084864 spec.py:346] Evaluating on the validation split. +I0915 03:35:01.880457 140069821084864 spec.py:363] Evaluating on the test split. +I0915 03:38:35.301773 140069821084864 submission_runner.py:516] Time since start: 36552.16s, Step: 7080, {'train/loss': 0.12131296080331382, 'validation/loss': 0.12381045709418861, 'validation/num_examples': 83274637, 'test/loss': 0.12617998732556795, 'test/num_examples': 95000000, 'score': 12134.730668783188, 'total_duration': 36552.16082024574, 'accumulated_submission_time': 12134.730668783188, 'accumulated_eval_time': 24392.45693564415, 'accumulated_logging_time': 1.1855676174163818} +I0915 03:38:35.324617 140026536982272 logging_writer.py:48] [7080] accumulated_eval_time=24392.5, accumulated_logging_time=1.18557, accumulated_submission_time=12134.7, global_step=7080, preemption_count=0, score=12134.7, test/loss=0.12618, test/num_examples=95000000, total_duration=36552.2, train/loss=0.121313, validation/loss=0.12381, validation/num_examples=83274637 +I0915 03:44:31.990007 140069821084864 spec.py:333] Evaluating on the training split. +I0915 03:45:30.481608 140069821084864 spec.py:346] Evaluating on the validation split. +I0915 03:48:28.892586 140069821084864 spec.py:363] Evaluating on the test split. +I0915 03:52:02.853458 140069821084864 submission_runner.py:516] Time since start: 37359.71s, Step: 7319, {'train/loss': 0.12137885767832389, 'validation/loss': 0.12387279992932258, 'validation/num_examples': 83274637, 'test/loss': 0.12630583417743885, 'test/num_examples': 95000000, 'score': 12490.695368289948, 'total_duration': 37359.71250939369, 'accumulated_submission_time': 12490.695368289948, 'accumulated_eval_time': 24843.320456027985, 'accumulated_logging_time': 1.2147247791290283} +I0915 03:52:02.876708 140026528589568 logging_writer.py:48] [7319] accumulated_eval_time=24843.3, accumulated_logging_time=1.21472, accumulated_submission_time=12490.7, global_step=7319, preemption_count=0, score=12490.7, test/loss=0.126306, test/num_examples=95000000, total_duration=37359.7, train/loss=0.121379, validation/loss=0.123873, validation/num_examples=83274637 +I0915 03:56:30.397613 140026536982272 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.0056297, loss=0.125352 +I0915 03:56:30.400846 140069821084864 submission.py:307] 7500) loss = 0.125, grad_norm = 0.006 +I0915 03:57:59.858484 140069821084864 spec.py:333] Evaluating on the training split. +I0915 03:58:58.351329 140069821084864 spec.py:346] Evaluating on the validation split. +I0915 04:01:57.338147 140069821084864 spec.py:363] Evaluating on the test split. +I0915 04:05:31.866446 140069821084864 submission_runner.py:516] Time since start: 38168.73s, Step: 7558, {'train/loss': 0.1215535438057609, 'validation/loss': 0.12387690687957276, 'validation/num_examples': 83274637, 'test/loss': 0.12637291547803378, 'test/num_examples': 95000000, 'score': 12846.98413515091, 'total_duration': 38168.725474357605, 'accumulated_submission_time': 12846.98413515091, 'accumulated_eval_time': 25295.328444242477, 'accumulated_logging_time': 1.2445180416107178} +I0915 04:05:31.891374 140026528589568 logging_writer.py:48] [7558] accumulated_eval_time=25295.3, accumulated_logging_time=1.24452, accumulated_submission_time=12847, global_step=7558, preemption_count=0, score=12847, test/loss=0.126373, test/num_examples=95000000, total_duration=38168.7, train/loss=0.121554, validation/loss=0.123877, validation/num_examples=83274637 +I0915 04:11:28.658485 140069821084864 spec.py:333] Evaluating on the training split. +I0915 04:12:27.040307 140069821084864 spec.py:346] Evaluating on the validation split. +I0915 04:15:25.922105 140069821084864 spec.py:363] Evaluating on the test split. +I0915 04:18:59.744206 140069821084864 submission_runner.py:516] Time since start: 38976.60s, Step: 7778, {'train/loss': 0.12235634997831528, 'validation/loss': 0.12383803090201297, 'validation/num_examples': 83274637, 'test/loss': 0.12624362888729698, 'test/num_examples': 95000000, 'score': 13203.063688755035, 'total_duration': 38976.60324835777, 'accumulated_submission_time': 13203.063688755035, 'accumulated_eval_time': 25746.41421842575, 'accumulated_logging_time': 1.2761409282684326} +I0915 04:18:59.769758 140026536982272 logging_writer.py:48] [7778] accumulated_eval_time=25746.4, accumulated_logging_time=1.27614, accumulated_submission_time=13203.1, global_step=7778, preemption_count=0, score=13203.1, test/loss=0.126244, test/num_examples=95000000, total_duration=38976.6, train/loss=0.122356, validation/loss=0.123838, validation/num_examples=83274637 +I0915 04:24:56.963710 140026528589568 logging_writer.py:48] [7988] global_step=7988, preemption_count=0, score=13559.8 +I0915 04:25:05.272166 140069821084864 submission_runner.py:857] Final criteo1tb score: 13559.849144935608 diff --git a/logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..d77bbb25 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,39 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/loss,test/num_examples,total_duration,train/loss,validation/loss,validation/num_examples +2144.502624988556,0.0,11.428372621536257,1,0,11.428372621536257,0.6891419952508223,95000000,2156.258265018463,0.6890505460885573,0.6889704567631297,83274637 +3239.08166217804,0.1761083602905273,367.842592716217,186,0,367.842592716217,0.1316768300437525,95000000,3608.1425683498383,0.1269467330458767,0.1290675938772927,83274637 +4376.664420127869,0.2049825191497802,724.8272361755371,370,0,724.8272361755371,0.130261814970157,95000000,5103.454279661179,0.1282661345811348,0.1275410413946764,83274637 +5472.717729330063,0.2337865829467773,1082.1323177814484,553,0,1082.1323177814484,0.1293224617691843,95000000,6557.551578044891,0.1267044816886962,0.1268569404440916,83274637 +6553.85037279129,0.2626612186431885,1439.6988294124603,747,0,1439.6988294124603,0.129080506857621,95000000,7997.0038385391235,0.1263517817504705,0.1264811841288897,83274637 +7674.283470630646,0.2904345989227295,1795.705944776535,956,0,1795.705944776535,0.128750990815012,95000000,9474.198992490768,0.1243580790718835,0.1262111518070177,83274637 +8753.594065904617,0.332484483718872,2152.3411235809326,1171,0,2152.3411235809326,0.1290995803889224,95000000,10910.914236068726,0.1261888873957049,0.126377836313263,83274637 +9745.611643075945,0.3610024452209472,2508.20432639122,1358,0,2508.20432639122,0.1283687012205425,95000000,12259.538949489594,0.1232844286442708,0.1261147570713908,83274637 +10786.024005651474,0.3887231349945068,2864.1562666893005,1549,0,2864.1562666893005,0.127942265158081,95000000,13656.649791002274,0.125281200880896,0.1256871576798432,83274637 +11827.308275699615,0.4177336692810058,3220.04137635231,1748,0,3220.04137635231,0.1279195963399786,95000000,15054.550971746445,0.1253406900636589,0.1254944849850555,83274637 +12770.925372123718,0.4457502365112304,3576.227335691452,1953,0,3576.227335691452,0.1276889075049149,95000000,16355.084664106367,0.12294707671647,0.1253609903469546,83274637 +13567.52386879921,0.4758825302124023,3932.574423313141,2167,0,3932.574423313141,0.1280087134473298,95000000,17508.73826622963,0.1231787101783924,0.1255625145055109,83274637 +14261.421571493149,0.504258394241333,4288.982084035873,2369,0,4288.982084035873,0.1277983086204127,95000000,18559.743125915527,0.1229076661910116,0.1253095672038426,83274637 +14877.79320716858,0.5324914455413818,4646.466147422791,2578,0,4646.466147422791,0.1271974353258635,95000000,19534.2995262146,0.123511217918211,0.1248087542796706,83274637 +15369.760269403458,0.5615274906158447,5003.59354186058,2781,0,5003.59354186058,0.1270975227237099,95000000,20384.09462237358,0.1244436629439374,0.1247337566701084,83274637 +15818.821569442747,0.5901992321014404,5360.208089351654,2976,0,5360.208089351654,0.1271950413490697,95000000,21190.468819379807,0.1219188942123785,0.1247836267081731,83274637 +16267.482496500015,0.6184685230255127,5715.89444732666,3204,0,5715.89444732666,0.1270577581971821,95000000,21995.53031945229,0.1232015813627382,0.1246898326649552,83274637 +16718.54413294792,0.6468093395233154,6073.568929672241,3402,0,6073.568929672241,0.1271449026751066,95000000,22804.96911668777,0.1240498279854391,0.1247071737290302,83274637 +17170.37621164322,0.6755161285400391,6429.9725069999695,3612,0,6429.9725069999695,0.1271164032991108,95000000,23613.909635543823,0.1219526845802405,0.1246549948902484,83274637 +17620.24062728882,0.7046818733215332,6786.63451218605,3813,0,6786.63451218605,0.1271545092016922,95000000,24421.138518333435,0.123607654512015,0.1246874817967047,83274637 +18071.862988710403,0.7509946823120117,7143.305164813995,4010,0,7143.305164813995,0.1270927432146574,95000000,25230.1517534256,0.1233025302345707,0.1246869484784929,83274637 +18522.95866727829,0.7810301780700684,7499.636653900146,4238,0,7499.636653900146,0.126830500844855,95000000,26038.29193878174,0.1229911493668825,0.1244719247318919,83274637 +18976.23560285568,0.8097052574157715,7855.612186908722,4450,0,7855.612186908722,0.1267312048434609,95000000,26848.252278089523,0.1231869413643289,0.1244249022823069,83274637 +19427.10549354553,0.8389811515808105,8212.401973485947,4655,0,8212.401973485947,0.1268196325547067,95000000,27656.617559194565,0.1242115630530232,0.1244635887271983,83274637 +19878.17617082596,0.868128776550293,8569.17642068863,4872,0,8569.17642068863,0.1266996562864605,95000000,28465.17506074905,0.1233847696990428,0.1244418440649625,83274637 +20330.358067274094,0.8973524570465088,8925.319944858551,5083,0,8925.319944858551,0.1265404695667467,95000000,29274.210667848587,0.1204879265441646,0.1243024344718092,83274637 +20781.310081481934,0.92828631401062,9282.30319094658,5295,0,9282.30319094658,0.1266514545157984,95000000,30082.853084802628,0.1228923975745996,0.1242602879709712,83274637 +21233.65428447724,0.9595811367034912,9639.686212778091,5510,0,9639.686212778091,0.1266076050482499,95000000,30893.29542040825,0.1226458123943909,0.1242426659513741,83274637 +21685.651729106903,1.004307985305786,9996.833044290544,5735,0,9996.833044290544,0.1265822443932784,95000000,31703.16938185692,0.1205492419753749,0.1241994243948032,83274637 +22137.59357309341,1.034571409225464,10353.056209802628,5955,0,10353.056209802628,0.1262972319644325,95000000,32512.04721045494,0.1202869151170894,0.1239516846263875,83274637 +22589.660952091217,1.0645341873168943,10708.771770477297,6175,0,10708.771770477297,0.1265060532957378,95000000,33320.54216980934,0.1221755391485899,0.1241242689741346,83274637 +23039.951890945435,1.094493389129639,11065.027774333954,6388,0,11065.027774333954,0.1262167004445929,95000000,34127.79482841492,0.1223595607311219,0.1239190742885701,83274637 +23490.75907850265,1.1259403228759766,11421.166724920273,6628,0,11421.166724920273,0.1261504045039126,95000000,34935.46488428116,0.1217174865475821,0.1238195275884534,83274637 +23941.831956148148,1.155168533325195,11778.461151599884,6864,0,11778.461151599884,0.1260945113824141,95000000,35744.55286478996,0.1214620788683724,0.1237763680963989,83274637 +24392.45693564415,1.1855676174163818,12134.730668783188,7080,0,12134.730668783188,0.1261799873255679,95000000,36552.16082024574,0.1213129608033138,0.1238104570941886,83274637 +24843.320456027985,1.2147247791290283,12490.695368289948,7319,0,12490.695368289948,0.1263058341774388,95000000,37359.71250939369,0.1213788576783238,0.1238727999293225,83274637 +25295.32844424248,1.2445180416107178,12846.98413515091,7558,0,12846.98413515091,0.1263729154780337,95000000,38168.725474357605,0.1215535438057609,0.1238769068795727,83274637 +25746.41421842575,1.2761409282684326,13203.063688755035,7778,0,13203.063688755035,0.12624362888729698,95000000,38976.60324835777,0.12235634997831528,0.12383803090201297,83274637 diff --git a/logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/trial_1/events.out.tfevents.1789406963.daf5ed747178.38.0.v2 b/logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/trial_1/events.out.tfevents.1789406963.daf5ed747178.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..6262a1b054caf3af69f84126a7a1dfdeb57cb32e GIT binary patch literal 57482 zcmbWAd0b3w|Hl=j#garJM6{PSNkZ0=D2bA#LXSGFo2RedxMl9X0cM8w17!LKxce^k^IzcY7o<1O&);1yFu>DW&%OD-FLep>_Y83N zFVAk$)cKp%cp`tCbWerEtIcUz{bc&ewDa~`x^%RRl$d;*{_U*&^ha9D%1Fz!Upmjt z+s%LJ6(t>8gTJ%;*R89QcIx|gR!wGrjPyc(xA`u;{PI=|IZNdCgP2)@!yC!MvAkfU6)lCcl_V_ z<;1kqWd_Q~1Tm4Z_oK|0iJ#s zG{=<|6cjnFVbtHCHYwLqrB`mGf{bMhWfcEkkZ+)mi~DjnpT*wp)D`wGin9T_*?lWw z-~xTxP6hI=ae({M0D#-)MO)LPnVlXF`@5!|s>fzYZ`ngd=*j5Fbm2v?ttP|pEUxC? z%5kd({f+XO-P=yO_#hS4Uq(enVrlbD_TN_RFEtqjFv{zlcANekPgwDfqjdiiYFt}J zQ>IgZU%>yidw!rl?^6Gl(?3UY9OUdseP%W8Z_b+jEPJyw|VmdeFDAR z0^H}jEDfCNM_i%e2Fn=9DEc#@r~Iv z(snngL~dI`&{&VMCAMaNL*%1hS4sbKmkQC9(Uy^0?Cx{|+dIrBV^x6j?~o)@p(&P>i~?fKSn$Nny2A0V(dk z^LzjlXLG1&b_&KFb>aE$a-i6gPepK3R3OE}C(>VlBE3G4Pa(ymSoTR4DU9akJ_d@< zjnuS|6io{I`vFDrFDinYq75GYz2iF{1Bx&0!}t`^Op1Qn#gSssElo|JP?e*m*(n(3 zMV_X%4^T`~q9V8{7Fe=tyJdr;FtQe1ag^am);E~X;5 zDfE$IOnI0TP!#Mr&8O(Vq_}6EhZN&a=C}dHx&&&Po#H$F)LYkKS2|F8Fkb&I|bv83er294HWY}QW4w~c1RH&_T@29 zlvqCCQ*>fd=&d@46lEGO^MJzLj+$ntVC+SH@34nJQRPHMa8tMRdPvJEMYb>I3CC~O0$X?BVx`W?0XhxY)W&|O7Ea8oQsit|UOH37x`%ME-AIVOc( zyAY&k$dkjp2)#i~3rTV6z>Y+qNWV)(a8raJ#rL`{6M*7w&2K(M7bZo$n>$kY%cS># zy$EZdrr9aJ(Dy<<>bEgasD7g&xG5r#Vri#ti-AJBd#5M7cS-V0ijQn?bMI=%j&P`PZibT8Ke2T72iubN&ND>?okPd}A1&q8pQ9NnbUjh+n)L_rl{IHO)?8ki8Nk!oF#~1d8a# zR0KCgGE&^S_5LhSy!d0yr|8b47?RizmoP)`Wf<&5s?@~)rr9Z0YMEffo!QUujxz2- zMQ~H3Aw^vGkZPb9ukOrWha!_ACwl-cq1&5S2cTG>PffE^$p1Nk5t|&|J%GJPHl-rC zDKd~^NxJWJpg29xlTV?(vBeZ+|b}E9Kq5>)0hwJ`;y^!|($fxMd zq!2&T1DB92VH5{@;k}HSW~azC?2Zw~@@+oDUZk(4BDg7Pk>Wx~oHJ154BTAGdmYt> zNwIjg5-#EAv21*=H%gD1W~Vr4S2`R%VqNq>whK@sn@|zl6z`E@`2ErcKv5dm? zFO%YxOi!e^qc;j)nQxD!rr9a%(v&e`um3|l^|~LTBDg8O(I~!WsQ|^O)$x1^6(+@W zHQHXBtNtYhcT{6IHO)@3_r?gMcv^b{KgkK*Mn!N_v>6Q>I`D}Ko)rf7#08M)EAuoq*mCGjcxGbuJ4 z>WWLa>vkg&C?4FVrr9Z;cR7V#^wC|=4k!#BQW4w~Qb^&paf1a=6izWNu-k zMcvN_irVkgG&_Y{lp#jw9XK%>C_>xb{BHr=6mm$>G<6cbrd<^?x16^Z1DO=F{&c}5 zBo0+nxvw&ilSi*k`;HDUW6viI$AAn+AxkV*!FSM8x%Cgjy&Qd9&tg(*^$J#)psCAnXJ&tJzNCdIY~9ntQ<&f=$l!Zn|oW~X?wju!F0 zYnxc0=v72Ta8sBg#qodIy2DNX&3TATZg4U@y(Q)W~Xp$qP+q!`_i}w zC?=1hBDg8WBE|2?W;w7ID)$$>2jd3__tFHkHUyavFMO2Qrs%Lqy`idqo`?iik@;=NFnc1dju$E z?x7;MDQu8p!naiqfueZQ9zKOJlcL*n8b#fbOA$b^+mV`Prx-V>J5qR+bi(uEq6-zl zO<{)=TWU*B0Y$^{;jegmF_=kFex63LMe;Xx)VK_4nw?@q;6 zDZ;fsY=*s9(MPI|M`6mOD4a&4sF2bx1&TOzYMPy*kL+>W3%O-my@5hipNin7a7Bur zzcQ8qg;wegKE+Tbh0T`sct@#incoH|I-jAY*(t`&7>IjevE_Pyppd^rMQ~HhM~VuQ z&0m4y?kTm`yuC1EQoJgZL5fO^ZuUT-c#)cBr;zz5h7_KEg7L4D4%ev&ZVC^iICiUd zJD}Jz^CF*O7?a}c3>t-o+q;)QF=swC%}&vDmG;U!+u*b6srDuSEB8!0SL4KxOd zHbWiW@bsubhNG8R`eYCwunbG+sP{is`)9e(98w-)5(T4VI`BTPJ1UJPhq^P*HaUxLkT6>yL zVZo&MGKaPo$rdXY14Z|Z)HFMVlWPuAWS=v@SLTLMR0KCg2vWQ#E9(Rlk@3dwczeO} zFSlyf&?tguIky7}rxa?Monmd<2K2(uNTUhvsMJ$b1UE$}QuxKm#{fmbtUG*)(agPg ze4f@(8l_K7}QdLg^E2FX~5I zm;%L>&(t(KMb+LrxEH!_Z1I!-bkBaWZ(^C*EFIGbuv9)4Z@pXkg@qW6!ro=t44{xZMNP9) zc%2Brz4)~8zBo_}zd%KBQ|v*Cv5L8duotI4#qlX7Gb!>n($4NF$CtYR#nYeEG&@CR zj})Z1B&9M4DDv7J_-_H+6#J0EgfcI4q-K+#8unr5fy zYdsk$PB=aA0*Y`IDuSCL4k^Cfi(CYIG4<3{K7}2V!di+(p&(zm4k$WjQq$}dX)B}A zi}ibt7y-qvY$}4AA|5F^ci43kDAY_QedX=N6efj@zIoQ`j>pEFaPKqA>OB1fW=b9bY%${XQrmt& z@za!w;HEf+6eYWr|AoDHoqn57;mD+DQ$eFRAl_95C=Ontrr9YXX80gQSFx>9K(Ql- zir}U=ffTE+*5G&kdnjA|;O&JIlVbY`8pRz6wQE4}V*oYHP7zgXhZNI}M!o`y5*;dn zn<5P7cpTD^7!xd?mAHs+!Uvg z;<1Y5XrSn7x#TBrFPxbaHTkrTn(*w|57>)%TWXq};_dfLw0qga>Hol9oN=TgxGByc zMZ4cSrvgPE-|u{inM?}3yR?onPB$$8inc4LX?BX8_UDnJV*8y5K(RlRir}U=hZGA2 zI-wW4Ya)K}_F@*3VnjUc)T@3{Xbu!E4b(I{MJKa&NRjmGL^Mz={Yph}Q)D2;g{o=Y zVJ|+E^!m-Cn9Zc<-AMZq$2YOd_G!LwXRDs+);%`s0eO~D@YNa+Yviz z@1H#WI$W6)PLsv(Lf)RUA_OQ#NZtN#nw?^=zdA-(ou0l0DDKNq5!@76NHHU7MLO(7 zX{BeIPR-A4b7N9;&!bToDcE%Yim=zzG&{xI`gKTgNL}+FP(**EBDg88A%*F#{JlU? z#RHA5$v~0mK}B#= zyX2BY-j!xN)6n`c?a!A3^Tzm^|9$w}t@8=?(n|M8hKLU}?5uzKsQ%xq=+6Z%()vXt zi}_T;(l_VFz&EjmT=qT;505fkf>A7D>B)2$bxDOj#7AL}UF5CPfDWW~w&?DFi|3P|W{E-Z~W(Plt zb3Ihl$cgqc`k2%~18}hSQ4%7^!9IIv$HdI(*9^eH2d79_&b9nvOD;{ zN8{kGX>NZZ0}e`Okqi+XbW9IP0tav0Bw0)cKNt@l3l0vJ?`#MTdfX-R0vtRPd;st6 zX!YX>;Gp$G5+tmHjOAUJl-(5^EPg^#TXt~DjR;(&O%KO*;Gj_j$rH-K_cc>+H}kHn z90v}%z9Ny_4qiY98-gSUfrGm5NQ!6b#x<1WG|CmKD3t^iBC-Z zpiGASAQ>W-$@}BQQcxxb#Pp~O&MXu2u^X^VbbeoRfHJX_BJ%>uWKn%SYC5%J<4GtJ z<4z<<*fL=(&)ENgEtE;70!eMTOs+_z;wsOGdrW{b>DY_p2~{Rp?tWM%Ny9UAK%9U2 zlSoc+EM#yWv{wA13T2Y6Nm4{Blh8#;_~a$iOER0gHaqx6g3H0#b7|(7oU0uU4i?@d zA%Yy7u%4DQx{EY!NFBUB$3_06S_19@5;yQ0|!l?kqi+Xl-s57 zFF2T1MY5O<&i;~)dw+5D{G;ID=sGelz(J|nCcL|Q89ki@4o+_%LBcx7SYGEXZQQ}Z z)1OId%MRYVM{A0S>qex5gVH}qo=^@(1c#%8PudhL0S9}um8EWZP6xBha33B=hU$QW zi=|15Xb#3Irq;kY__e>&FNJ0YFSX}#aOuh4_%2<&&UroCx+--b#(+COe4|n*X0c4#0MRi$~DrUsh8Bd05;;ct9xUb1dye1#^ zC}csIeCZH2zk4&t4r#73Nf|<`fNnuf_n=H7yOR(>Wn$n-OR7v+wh#_2CK)1@$>UR#ot#nFNN>eh(x6=aU6c zCfTNBUO<^72Ww%OJbY;^0cBD(f&>X$CXD6DiaqKFWnw>$q_$ipt~aG{mBV7%OGBAN zP9%9km5Il&qgW=LieDT6ac*#Me@Te}|_2@c-sK$2SJV0*od|A2$0T}dLl zgWZ?X4)mFIn;gMGy`CgPLurUD#|EQ2Gri0_IE8_{LzBefZ91K$@^8y^y*g6vL z?!qkHm*C)OT@oa$gN)^s2KGJ$4n8*`sVzI`KVAt}`B1N8Jvg|;jN}RBpj}Z0Iyh6Z zrz$w;GKxfUI+&G$4h|9%KMoGoS&|gd9JGF@kOv3)Izzk9JO`h#Jj(x2N7`q&kEibb z2zR)vIT>d^&@I0W!oPZzXy;9ZYtnu!$>6*uAs6wQcyBjs4+nasMx%U><}%sRfvZf; zPoRBUBKrD|RZu3A3`vNfGUR;zuZxVGd+oK$*nOorano51EMX z4&nb=0fy#aCJ zeMlsyIPuvij*r)%3@8(a0FokFnW%3T!)J31%kf#?%}=aXTpJ=`mXx!Wns!1 z;Go-2lG?I^=66kTm0e=LJAs3D#eJ#!Qz!@ZD~r%Uol#e+!NDA963OXcNGUov+$^9y zIA||NQbco5)g>LDPV=l&2u7J3a>2zy zaBz$YNotjY2N(U>2@W1sBZ=$|R%Ou~+;H-s0yr40OEN@suza=QC2(-8G09>&n6SssQd4$!d-nlnq6b#{h3JmWYSXMa)sPN6TpsE+&5?*?3x$!AFh=QU~b9k0oS z-YymJqFO4qsYtoGOuou-l}X=mw0G#zdb7GfnWPtx5J6?KYzOVa#PoT$49a9~2}x>I znM_=`atmDQd*vjNy-dz4&|XyUAF5siWfJ<5WQbTM^)h3upiBnWlPqSL7~}v$Cn=(p$>yw}&)}d% zx6Eg~n;p!S=W?*!dD?+K{PlGka4@?M2@&L=`V^XjV?SO>0tW}Hk)&2R7*TXY85~@y zOA^@~+*U?A&_gbac?u4WH6|G%I+(Y_qXrz@Yeuq|4nA!kj(czQRx1x2^d3d#1vt3s zPbA*mnF~@@fP+IVNszD(GM1NTXxjrEte8YnTXrz5Xb!G&LwI~FI5@ubKj7qIsVrwW>^p?J7=zOI=b#64}c{ z)`C_h-#lwdpiIo4kqi;bWZ-STWGIvDDw4%4lZ!^%@g_dKK4CkQN$)x`FQ80f_r;*5 zV?z&Pnf!i7f`ly-#`3%rcAG(&O!-VwTP_owAP-z+eE6;tP$tWMkUXKvWY!cZER!XR zG-E-W)#7ugTb@%Km43Jn;-@?OfHHBECMlwoi9#R0-r!*Do`S@_&1F*7jmts%XS7QB zd`I#%IB1keLIgSJ89;L|Vt(>LaL_%KB(=)H@&yrc;GptplF060=|Gx;31v?9;9%Da zBtt|8`)Z_h1P3Fpk}Rf!S+n-x-luIFfq#0weS^#ka8Tdk0N&ku#nUgqfu3=P1PSXP zV|mvfuki;5l?zE~%MQw_FUD0$+Sh#o2Te;zo=^^s&FYK}9?X*P0|&P~Cy|^E7U<$W zXvPFh1qT~mk`&P#^gU&@5ge3T>EWx=>|neimxKG$n(*|RXgz%}ICw6Mga~qQ+g6%` zzWH-sfP<^IlB8BSICHdLJ~(K*izKo;=rxk&V1J7WYj99Jj%0}F;D-aZ$AW`{l1LWQ z!J})B;NE-9ELOyG@EDmF;NbhsNqBd+9dXeT9DH}01PSXPV|iM>x>LYG)l8DwvV#+} zgK(8OGnT1>gTC1$Pbdd%6cy3ICHldp;GpYm63OXc8#CO8hvL8E!NJN0BtpNBfarFPgx64}edw->EU^g~lkpiGo^kqi;bBwW(s4wOl+{UnQ7CK?A$ z1(b8{3A zrry}|63S%DIg%$-nKVxAgJt6NcE25n6LFbDa*AVNh5L}!Q}GOxN%3`(B3hXkUAT4( z9PAJ!CEmZ;!P`B#9DLl7_M+P4?5rc;;G;bxM393k7tkCmzu)v0927f5l3L~9%5S3v zgM-_WNg}(0;|*yJ=9lNL1P6PZBpD()s5ZTuG&mT2mSiy<+>(D8_x_YY<{xnI?L{&# zz`^&Lr}6H#=~HV94$5C6LBcx7SYE~Lop#`0!!459vV)5ILvfXnWd`^!E_3daJfR%4 zi5`d!UU>D?8XP?Om_%|q81I1lAU)h<95~qYjHHO>;EUm>mcxPms`ol|o`c6&eztk6 zA?@XKnb@{*aECupC*$k~daidp9_Wrk657Bu5i=keoYy4F8Lvs)&aVMbCaL*j^LZfq zl)1{p;TjFc`*3VmD3gjJ5+bNf`b?yiNo8&U9_X{nNK&iH#8o~07hGz!YLduaCU>=H zWfD1Z#5O3C*RM&2h-Fe6TyP%BB=!TzVwTCRhFiE{9bDR`L75EtM&<>SNnvmXYI^U= z%O6lC?S7LWVatTEyv;7--$I#Kwi`#?Pc4^8_JD9)<@@Ikx50saScc>YRVMN8bg)bm z2WMRZaU#2rNN#axPoNl|?H>VU@?4Rmh*l;wS_kk`GU-hUyHuMG^q0N49Bixhi}nuv z%7SumP&1l@2y#$;7R^D)SN3smpnL5jNv(2ld7$c1aByq_No05Mjxo)_8{c(~gM;5v zNQQ_G_E^2Y2prT+BUwxbyLNko4$hfzFa;cxJx}HZIB0qM3f|qDwS8`YgZD3yAYmP3 zEN^SIg)unT{W?i)*}>rCe{q#6o9vXq!4-KVPbdea78s+0mD^Ry!NL3j63OYHoiFag ziDMb_z(MC?k|LUe9WvIY!h!DT8~U2(U_8t3&F@+I1?{UHzZ$<>5fwzn*$?!jV^i?Y zUdF$@JHs_8Swk{7uSuJgcuk^K72}U7$JjWy4rnfuqkXx`L^_=I4t@HZnyFAGX^tdB zP?DWa8$*Ty^eV_s@zv$X~`JGiwUmxFm#v=9BVBMvr#gGysbh#&`5-_YLcKe+kw zD>xWCktDUs!7G^>L%_k=_9T(r!Kcq@{|4>wo@1}U!BQuZA)!CfvSi|L@! z?APd^ovWiGI4HS*%nNWZ+WQXP-GP^$SA&Dn-Xus^2N}y-sP;M^9CY<3sVzJB<}B^o zHX7D?ufV~uqh~{9F`%wIO zf(e0r&O&$#!i?;Sb~4i>&7^8y@P zoAm%49N8H38XSD`h6D-gAY*ywY(rGR!4n@zYRe9;u-J*K%q)~b2j!bco=^^!SlOb3 z1N9fo1P3R!c}m^#oDRxu!hPso<$;eesgfiqqB&UW_h~r1sLt+vN=&W!9NekOe9Y*8 z^|TXkfQCr~+~EppWSsp)b(HxT{FJO`&yQ(vO(Jzk2In=2-;CEp(`Kmxd`dQ^=f^8N zkdp(r%H+^oS_OP8{S*sj;xT}P2r84D8d}nh>em0$@V0*Q=H|r@ARp>oSF{@`ke(NMYJ-Rm)!|}=Ql5}+rR3~WfGyr<)Hd0+JU~e zleZl>==G3<2y$@FQ<{Uf_qfIR>qqb@dJ$==Z*pAYmP3 zEN@1FSx0ctRjfO8Keg@Ly{)zS}+1q_)&)^Qv)FtEW2fBsT z0sNHgx!cJ1a7|(clML=_Li?WM^xWg$;6Q(=?KD-RxlEp_bCrqfH5yJvWv}T_CNl<; z5J6=!K8jW*n>6JvLYX9(lcZLa$=H%I8@SXiV@M)!-IsYgQS1EO1;{hlW^F<_2s4}U}aK|$FY2>^W#PM80B00q=iNi8k)=;w^ z%H&B9NfE6~uIV>z00)C7ukEeb?BGdFE(gU=(%vK3`JQM84k|g55J3*=t)e-&UMhG6 zIQY_yB(=)H$SRi;;9!ymNo05MO&ZO?_8O*9;Gl#b$q>;&>yX1?;NU-jB#Y^wiF6md zDHppf{S6M51(SIJ4xW{GgF(A5e~APKdxer9VI5>F?{}fb18{In1W9e#!2`~RaFu(O zT<-u5?%hiAgmO?P(FYxL_H*5V1_MyWPU)v4q5tEM}Qpjq8qfE$o+u?+(n9 z$h?3u+3oxRgJQ@3*bil*mP&$zEfdD_tVYWyLYYLJBB?ExiR}y8ckw$ce0LPe#3h5| z2~{S_M*^`-QnJR3263)kC6S!s=pVx}2@p@}17)(~CP@*kOzQpn{eS~KD>(YJR`Y@W zN{7qAuqxV%>JK+J{{aVQgp&|K4vw2ebI`eJ;U94D#TJs(DhJbtmxq9Z4|bA7b_a9g zXbxT-llKc8T)vlNi0EMb75!>B(61aMSxg6~87ZSW2c}(?h68<2GMN|PpyP?J7*uz3 z_)c(e>j@Ghtb>f@4c+2^&trLchNQOa;AF29T;-HViKlSFn0DZYRO^}B&>ss<-M=j zZvf}uhBA`cvV;1cj^Qfp-rp+(2eqn6o=^@x_Fj(;zV~yD1_wvKA(5O8DrKUBU&<O8k^z=2l~1&5+rPyFqU^$ zw(DFdlOdZ)YRhF}ADo7`pC{VJK$*BkkvyTwBzbfMmWjM~3I5s3K9)psit{ZC#VNWO zw-L(Z^8u0~TA3vKNWTULtz$0B={7sqXu##*;oUR`)5dI80tbr^k`O@-PROGj=m`_l zhQonwl|qtQIXb#Gk7cT?{_nadcB09L?v2`stn0tw2F&$K{ z*1?-{$+_BaaIo+?nHS*T9a{->@Yshg=fS}qw@Hw&4l4;=jcnB)oN;4G!h=wOw(%py3vo|lnGP6y+2(LtNL!OOrw&uWq)nuFa26t9K@ zUB&g|V4j23hRg$fO4cVl0d?lQ#b0qv@+9Nz2l^5>Up&xzbn&=U~Y#C=)e*k|AQ5{QJWbzZAD(Imu#{Nos&0+LdYR6#!+T zvxdwID3hQdDJ+xFZOLn)OqPa`AYsddu{>A5;sz*_SDQ#`%ViQ4b{=uRma3*enG{8l zJfX^@pid;0iLHJ|FA!(w9umnd&V3Ze=-fzaD3hjmk|J7}94K9>1P6MV^t(j8<^#Q8 zFqebM+262Ij3m3`&+&BcNrMwA~8_dc7XWVmeqNO?#AfQhEC*aL|1)nHS*T)_7TTaD{HVC1tM=-}4Pck;kN zd0P_6>0roXbTD|76aMqL zt9ax)sJTp%hj5k2JAYc4bP8Xe0SEe#IuatNOr}TD$|UFTKOs;ix$j9*tI8zoUPUOB z$&{}ok-bbD2Gh#q?(wd%P$rjukqi;bBzb|Q6O>6!+al_MGs~oY&v3M>U#Ukil!>(r znHNwdnzuS(nT(q+*$>LZOO6ByTPBREFCcmun&w_)qO)^aNn;qOe zl*_^3Xqtm+7tg*02Rm4h5J3(~o}xL}qscH99Q3jzNv(2ls>u=j9*f0flF07hl<71F zJ>1tdqJs`3LqrEl(r&tfg9bB67Sq97T4T_`;I!HY;NW#PGB3cva}8b4!E-bACxCCpvPgoLn7_P~c10;j`(5}1Iku#x8LZ^{=0cDb@)D6qTEAqlyC==V6BuLmY zVJxpE=B^i%Nr4+lZMjU6<7ofLJUYDW8iwKg% zba0f-WOQ)3*!$k#;LojOUVwvpER@he`vmo);9!Sn5+tmHjO9H(8fyy<_nu$*tjPhRx^TnGssdTVla;+Rvrze_PcSfbt8;JSR|m69TQw>Qn(h5l={pXh560 H^}q3dCTmFC literal 0 HcmV?d00001 diff --git a/logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/trial_1/flags_0.json b/logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/trial_1/flags_0.json new file mode 100644 index 00000000..011facee --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/trial_1/flags_0.json @@ -0,0 +1,62 @@ +{ + "logtostderr": false, + "alsologtostderr": false, + "log_dir": "", + "v": 0, + "verbosity": 0, + "logger_levels": {}, + "stderrthreshold": "fatal", + "showprefixforinfo": true, + "run_with_pdb": false, + "pdb_post_mortem": false, + "pdb": false, + "run_with_profiling": false, + "profile_file": null, + "use_cprofile_for_profiling": true, + "only_check_args": false, + "op_conversion_fallback_to_while_loop": true, + "delta_threshold": 0.5, + "tt_check_filter": false, + "tt_single_core_summaries": false, + "runtime_oom_exit": true, + "hbm_oom_exit": true, + "test_srcdir": "", + "test_tmpdir": "/tmp/absl_testing", + "test_random_seed": 301, + "test_randomize_ordering_seed": "", + "xml_output_file": "", + "experimental_orbax_use_distributed_process_id": false, + "experimental_orbax_use_distributed_barrier": false, + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "workload": "criteo1tb", + "tuning_ruleset": "self", + "tuning_search_space": null, + "num_tuning_trials": 1, + "data_dir": "/data/criteo1tb", + "imagenet_v2_data_dir": null, + "librispeech_tokenizer_vocab_path": "", + "framework": "pytorch", + "torch_compile": true, + "experiment_dir": "/experiment_runs", + "experiment_name": "submissions_a100/ademamix_golden/study_0", + "save_checkpoints": false, + "save_intermediate_checkpoints": true, + "resume_last_run": null, + "append_timestamp": false, + "use_wandb": false, + "profile": false, + "max_global_steps": null, + "overwrite": true, + "hparam_start_index": null, + "hparam_end_index": null, + "rng_seed": -858678099, + "set_pytorch_max_split_size": false, + "pytorch_eval_num_workers": 0, + "capture_jax_trace": false, + "skip_evals": false, + "?": false, + "help": false, + "helpshort": false, + "helpfull": false, + "helpxml": false +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/trial_1/measurements.csv b/logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/trial_1/measurements.csv new file mode 100644 index 00000000..03a5e553 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/trial_1/measurements.csv @@ -0,0 +1,156 @@ +global_step,grad_norm,loss,train/loss,validation/loss,validation/num_examples,test/loss,test/num_examples,score,total_duration,accumulated_submission_time,accumulated_eval_time,accumulated_logging_time,preemption_count +0,0.4999999701976776,0.6891981363296509,,,,,,,,,,, +1,,,0.6890505460885573,0.6889704567631297,83274637.0,0.6891419952508223,95000000.0,11.428372621536257,2156.258265018463,11.428372621536257,2144.502624988556,0.0,0.0 +1,0.4999999403953552,0.6891802549362183,,,,,,,,,,, +2,0.4999999105930328,0.6837464570999146,,,,,,,,,,, +3,0.4999998807907104,0.6729466915130615,,,,,,,,,,, +4,0.4999999105930328,0.6568023562431335,,,,,,,,,,, +5,0.4999999105930328,0.6355364322662354,,,,,,,,,,, +6,0.4999999701976776,0.6098004579544067,,,,,,,,,,, +7,0.4999999105930328,0.5798840522766113,,,,,,,,,,, +8,0.4999999105930328,0.546683669090271,,,,,,,,,,, +9,0.4999999403953552,0.5100269913673401,,,,,,,,,,, +10,0.4999999105930328,0.4722042679786682,,,,,,,,,,, +11,0.4999999105930328,0.4325586259365082,,,,,,,,,,, +12,0.4999998211860657,0.3930420279502868,,,,,,,,,,, +13,0.4999999105930328,0.3533943295478821,,,,,,,,,,, +14,0.4999998211860657,0.3154005408287048,,,,,,,,,,, +15,0.4999997913837433,0.2799190282821655,,,,,,,,,,, +16,0.4999997913837433,0.2500318884849548,,,,,,,,,,, +17,0.4999997019767761,0.2240063548088073,,,,,,,,,,, +18,0.4999997019767761,0.2020479589700698,,,,,,,,,,, +19,0.4999994933605194,0.1820104122161865,,,,,,,,,,, +20,0.4999991059303283,0.1699636578559875,,,,,,,,,,, +21,0.2448775917291641,0.1622293144464492,,,,,,,,,,, +22,0.1778964102268219,0.1578529924154281,,,,,,,,,,, +23,0.3417897820472717,0.1608954071998596,,,,,,,,,,, +24,0.4483873844146728,0.1608816832304,,,,,,,,,,, +25,0.4927957057952881,0.1613159775733947,,,,,,,,,,, +26,0.484461635351181,0.1611492931842804,,,,,,,,,,, +27,0.3975645899772644,0.1576491892337799,,,,,,,,,,, +28,0.263543963432312,0.1516500115394592,,,,,,,,,,, +29,0.1466780751943588,0.1501007378101349,,,,,,,,,,, +30,0.1065990999341011,0.1487939357757568,,,,,,,,,,, +31,0.1530163288116455,0.150432214140892,,,,,,,,,,, +32,0.1863006800413131,0.1485837399959564,,,,,,,,,,, +33,0.1429472863674163,0.1470123827457428,,,,,,,,,,, +34,0.0864598676562309,0.1441746801137924,,,,,,,,,,, +35,0.10862348228693,0.143851026892662,,,,,,,,,,, +36,0.1743853092193603,0.1448598802089691,,,,,,,,,,, +37,0.1566678136587143,0.1425756812095642,,,,,,,,,,, +38,0.1257921457290649,0.1475099474191665,,,,,,,,,,, +39,0.059995137155056,0.1471109241247177,,,,,,,,,,, +40,0.0810338109731674,0.1484596729278564,,,,,,,,,,, +41,0.0790589153766632,0.1475858986377716,,,,,,,,,,, +42,0.046773687005043,0.1455501914024353,,,,,,,,,,, +43,0.0906158834695816,0.1450673043727874,,,,,,,,,,, +44,0.0859874933958053,0.1456043124198913,,,,,,,,,,, +45,0.0391920134425163,0.1463873237371444,,,,,,,,,,, +46,0.0361098237335681,0.1488816440105438,,,,,,,,,,, +47,0.1381985396146774,0.1384947150945663,,,,,,,,,,, +48,0.0569327324628829,0.1391542255878448,,,,,,,,,,, +49,0.1259511560201645,0.1383801102638244,,,,,,,,,,, +50,0.0325923413038253,0.1369504332542419,,,,,,,,,,, +51,0.0937807112932205,0.1363617479801178,,,,,,,,,,, +52,0.0366947837173938,0.1360705643892288,,,,,,,,,,, +53,0.091532215476036,0.1364435851573944,,,,,,,,,,, +54,0.0268381759524345,0.1347616016864776,,,,,,,,,,, +55,0.0731413438916206,0.1339487731456756,,,,,,,,,,, +56,0.0211234651505947,0.1341164410114288,,,,,,,,,,, +57,0.0403683371841907,0.1323749423027038,,,,,,,,,,, +58,0.0317249670624733,0.135570153594017,,,,,,,,,,, +59,0.0863036587834358,0.1351078301668167,,,,,,,,,,, +60,0.0498792566359043,0.1349466145038604,,,,,,,,,,, +61,0.025199607014656,0.1333629935979843,,,,,,,,,,, +62,0.0713535547256469,0.1315186023712158,,,,,,,,,,, +63,0.102803498506546,0.1334096491336822,,,,,,,,,,, +64,0.0958868861198425,0.1312216222286224,,,,,,,,,,, +65,0.0291764754801988,0.1313472986221313,,,,,,,,,,, +66,0.0735643804073333,0.1354669034481048,,,,,,,,,,, +67,0.1393014639616012,0.1364776194095611,,,,,,,,,,, +68,0.1323429346084594,0.134956881403923,,,,,,,,,,, +69,0.0660049691796302,0.134286031126976,,,,,,,,,,, +70,0.0209383536130189,0.1336187422275543,,,,,,,,,,, +71,0.0994184166193008,0.1347163915634155,,,,,,,,,,, +72,0.1882063001394271,0.1345784664154052,,,,,,,,,,, +73,0.267238974571228,0.1359064579010009,,,,,,,,,,, +74,0.2988440096378326,0.1337681114673614,,,,,,,,,,, +75,0.218725636601448,0.1334401369094848,,,,,,,,,,, +76,0.0166625026613473,0.13616743683815,,,,,,,,,,, +77,0.251330703496933,0.135746955871582,,,,,,,,,,, +78,0.3302744030952453,0.1352950930595398,,,,,,,,,,, +79,0.2014546245336532,0.1330580115318298,,,,,,,,,,, +80,0.018246978521347,0.1328441202640533,,,,,,,,,,, +81,0.1804081797599792,0.1340187489986419,,,,,,,,,,, +82,0.2469004392623901,0.1336222738027572,,,,,,,,,,, +83,0.1948887556791305,0.1294896304607391,,,,,,,,,,, +84,0.0518726259469985,0.129945695400238,,,,,,,,,,, +85,0.1132404133677482,0.1281684637069702,,,,,,,,,,, +86,0.2025592923164367,0.1272499561309814,,,,,,,,,,, +87,0.231786847114563,0.1278076171875,,,,,,,,,,, +88,0.2039473354816436,0.1276190429925918,,,,,,,,,,, +89,0.1355592161417007,0.1276016384363174,,,,,,,,,,, +90,0.0912989154458046,0.130664438009262,,,,,,,,,,, +91,0.0895129740238189,0.128712385892868,,,,,,,,,,, +92,0.1163216680288314,0.129517525434494,,,,,,,,,,, +93,0.2354787588119506,0.1259312182664871,,,,,,,,,,, +94,0.4493392705917358,0.1305947601795196,,,,,,,,,,, +95,0.4999993145465851,0.1298063546419143,,,,,,,,,,, +96,0.1635231226682663,0.1262754201889038,,,,,,,,,,, +97,0.257662832736969,0.1267436444759369,,,,,,,,,,, +98,0.4829124510288238,0.1278033852577209,,,,,,,,,,, +99,0.3951641917228699,0.1269267201423645,,,,,,,,,,, +100,0.0259142145514488,0.1229306757450103,,,,,,,,,,, +186,,,0.1269467330458767,0.1290675938772927,83274637.0,0.1316768300437525,95000000.0,367.842592716217,3608.1425683498383,367.842592716217,3239.08166217804,0.1761083602905273,0.0 +370,,,0.1282661345811348,0.1275410413946764,83274637.0,0.130261814970157,95000000.0,724.8272361755371,5103.454279661179,724.8272361755371,4376.664420127869,0.2049825191497802,0.0 +500,0.0315373055636882,0.1309966742992401,,,,,,,,,,, +553,,,0.1267044816886962,0.1268569404440916,83274637.0,0.1293224617691843,95000000.0,1082.1323177814484,6557.551578044891,1082.1323177814484,5472.717729330063,0.2337865829467773,0.0 +747,,,0.1263517817504705,0.1264811841288897,83274637.0,0.129080506857621,95000000.0,1439.6988294124603,7997.0038385391235,1439.6988294124603,6553.85037279129,0.2626612186431885,0.0 +956,,,0.1243580790718835,0.1262111518070177,83274637.0,0.128750990815012,95000000.0,1795.705944776535,9474.198992490768,1795.705944776535,7674.283470630646,0.2904345989227295,0.0 +1000,0.0400112755596637,0.1232190430164337,,,,,,,,,,, +1171,,,0.1261888873957049,0.126377836313263,83274637.0,0.1290995803889224,95000000.0,2152.3411235809326,10910.914236068726,2152.3411235809326,8753.594065904617,0.332484483718872,0.0 +1358,,,0.1232844286442708,0.1261147570713908,83274637.0,0.1283687012205425,95000000.0,2508.20432639122,12259.538949489594,2508.20432639122,9745.611643075945,0.3610024452209472,0.0 +1500,0.0107403462752699,0.1299348175525665,,,,,,,,,,, +1549,,,0.125281200880896,0.1256871576798432,83274637.0,0.127942265158081,95000000.0,2864.1562666893005,13656.649791002274,2864.1562666893005,10786.024005651474,0.3887231349945068,0.0 +1748,,,0.1253406900636589,0.1254944849850555,83274637.0,0.1279195963399786,95000000.0,3220.04137635231,15054.550971746445,3220.04137635231,11827.308275699615,0.4177336692810058,0.0 +1953,,,0.12294707671647,0.1253609903469546,83274637.0,0.1276889075049149,95000000.0,3576.227335691452,16355.084664106367,3576.227335691452,12770.925372123718,0.4457502365112304,0.0 +2000,0.0097095798701047,0.1257044672966003,,,,,,,,,,, +2167,,,0.1231787101783924,0.1255625145055109,83274637.0,0.1280087134473298,95000000.0,3932.574423313141,17508.73826622963,3932.574423313141,13567.52386879921,0.4758825302124023,0.0 +2369,,,0.1229076661910116,0.1253095672038426,83274637.0,0.1277983086204127,95000000.0,4288.982084035873,18559.743125915527,4288.982084035873,14261.421571493149,0.504258394241333,0.0 +2500,0.0150674227625131,0.1244876235723495,,,,,,,,,,, +2578,,,0.123511217918211,0.1248087542796706,83274637.0,0.1271974353258635,95000000.0,4646.466147422791,19534.2995262146,4646.466147422791,14877.79320716858,0.5324914455413818,0.0 +2781,,,0.1244436629439374,0.1247337566701084,83274637.0,0.1270975227237099,95000000.0,5003.59354186058,20384.09462237358,5003.59354186058,15369.760269403458,0.5615274906158447,0.0 +2976,,,0.1219188942123785,0.1247836267081731,83274637.0,0.1271950413490697,95000000.0,5360.208089351654,21190.468819379807,5360.208089351654,15818.821569442747,0.5901992321014404,0.0 +3000,0.0068174786865711,0.1229127794504165,,,,,,,,,,, +3204,,,0.1232015813627382,0.1246898326649552,83274637.0,0.1270577581971821,95000000.0,5715.89444732666,21995.53031945229,5715.89444732666,16267.482496500015,0.6184685230255127,0.0 +3402,,,0.1240498279854391,0.1247071737290302,83274637.0,0.1271449026751066,95000000.0,6073.568929672241,22804.96911668777,6073.568929672241,16718.54413294792,0.6468093395233154,0.0 +3500,0.0072151157073676,0.1216779053211212,,,,,,,,,,, +3612,,,0.1219526845802405,0.1246549948902484,83274637.0,0.1271164032991108,95000000.0,6429.9725069999695,23613.909635543823,6429.9725069999695,17170.37621164322,0.6755161285400391,0.0 +3813,,,0.123607654512015,0.1246874817967047,83274637.0,0.1271545092016922,95000000.0,6786.63451218605,24421.138518333435,6786.63451218605,17620.24062728882,0.7046818733215332,0.0 +4000,0.0127975093200802,0.1203376799821853,,,,,,,,,,, +4010,,,0.1233025302345707,0.1246869484784929,83274637.0,0.1270927432146574,95000000.0,7143.305164813995,25230.1517534256,7143.305164813995,18071.862988710403,0.7509946823120117,0.0 +4238,,,0.1229911493668825,0.1244719247318919,83274637.0,0.126830500844855,95000000.0,7499.636653900146,26038.29193878174,7499.636653900146,18522.95866727829,0.7810301780700684,0.0 +4450,,,0.1231869413643289,0.1244249022823069,83274637.0,0.1267312048434609,95000000.0,7855.612186908722,26848.252278089523,7855.612186908722,18976.23560285568,0.8097052574157715,0.0 +4500,0.0055003846064209,0.1228106319904327,,,,,,,,,,, +4655,,,0.1242115630530232,0.1244635887271983,83274637.0,0.1268196325547067,95000000.0,8212.401973485947,27656.617559194565,8212.401973485947,19427.10549354553,0.8389811515808105,0.0 +4872,,,0.1233847696990428,0.1244418440649625,83274637.0,0.1266996562864605,95000000.0,8569.17642068863,28465.17506074905,8569.17642068863,19878.17617082596,0.868128776550293,0.0 +5000,0.0194436311721801,0.1205823272466659,,,,,,,,,,, +5083,,,0.1204879265441646,0.1243024344718092,83274637.0,0.1265404695667467,95000000.0,8925.319944858551,29274.210667848587,8925.319944858551,20330.358067274094,0.8973524570465088,0.0 +5295,,,0.1228923975745996,0.1242602879709712,83274637.0,0.1266514545157984,95000000.0,9282.30319094658,30082.853084802628,9282.30319094658,20781.310081481934,0.92828631401062,0.0 +5500,0.0061199013143777,0.1254158169031143,,,,,,,,,,, +5510,,,0.1226458123943909,0.1242426659513741,83274637.0,0.1266076050482499,95000000.0,9639.686212778091,30893.29542040825,9639.686212778091,21233.65428447724,0.9595811367034912,0.0 +5735,,,0.1205492419753749,0.1241994243948032,83274637.0,0.1265822443932784,95000000.0,9996.833044290544,31703.16938185692,9996.833044290544,21685.651729106903,1.004307985305786,0.0 +5955,,,0.1202869151170894,0.1239516846263875,83274637.0,0.1262972319644325,95000000.0,10353.056209802628,32512.04721045494,10353.056209802628,22137.59357309341,1.034571409225464,0.0 +6000,0.0189948994666337,0.1220203042030334,,,,,,,,,,, +6175,,,0.1221755391485899,0.1241242689741346,83274637.0,0.1265060532957378,95000000.0,10708.771770477297,33320.54216980934,10708.771770477297,22589.660952091217,1.0645341873168943,0.0 +6388,,,0.1223595607311219,0.1239190742885701,83274637.0,0.1262167004445929,95000000.0,11065.027774333954,34127.79482841492,11065.027774333954,23039.951890945435,1.094493389129639,0.0 +6500,0.0145495990291237,0.1190191581845283,,,,,,,,,,, +6628,,,0.1217174865475821,0.1238195275884534,83274637.0,0.1261504045039126,95000000.0,11421.166724920273,34935.46488428116,11421.166724920273,23490.75907850265,1.1259403228759766,0.0 +6864,,,0.1214620788683724,0.1237763680963989,83274637.0,0.1260945113824141,95000000.0,11778.461151599884,35744.55286478996,11778.461151599884,23941.831956148148,1.155168533325195,0.0 +7000,0.0072777294553816,0.1230877265334129,,,,,,,,,,, +7080,,,0.1213129608033138,0.1238104570941886,83274637.0,0.1261799873255679,95000000.0,12134.730668783188,36552.16082024574,12134.730668783188,24392.45693564415,1.1855676174163818,0.0 +7319,,,0.1213788576783238,0.1238727999293225,83274637.0,0.1263058341774388,95000000.0,12490.695368289948,37359.71250939369,12490.695368289948,24843.320456027985,1.2147247791290283,0.0 +7500,0.0056297034025192,0.1253518164157867,,,,,,,,,,, +7558,,,0.1215535438057609,0.1238769068795727,83274637.0,0.1263729154780337,95000000.0,12846.98413515091,38168.725474357605,12846.98413515091,25295.32844424248,1.2445180416107178,0.0 +7778,,,0.1223563499783152,0.1238380309020129,83274637.0,0.1262436288872969,95000000.0,13203.063688755035,38976.60324835777,13203.063688755035,25746.41421842575,1.2761409282684326,0.0 +7988,,,,,,,,13559.849144935608,,,,,0.0 diff --git a/logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/trial_1/meta_data_0.json b/logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/trial_1/meta_data_0.json new file mode 100644 index 00000000..6899f8ac --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/criteo1tb_pytorch/trial_1/meta_data_0.json @@ -0,0 +1,72 @@ +{ + "workload.embed_dim": 128, + "workload.eval_batch_size": 8192, + "workload.eval_period_time_sec": 356, + "workload.max_allowed_runtime_sec": 8915, + "workload.num_dense_features": 13, + "workload.num_eval_train_examples": 83279872, + "workload.num_test_examples": 95000000, + "workload.num_train_examples": 4195197692, + "workload.num_validation_examples": 83274637, + "workload.step_hint": 10666, + "workload.target_metric_name": "loss", + "workload.test_target_value": 0.126041, + "workload.train_mean": 0.126041, + "workload.train_stddev": 0.126041, + "workload.use_layer_norm": false, + "workload.use_resnet": false, + "workload.validation_target_value": 0.123735, + "workload.vocab_size": 4194304, + "cpu.util.avg_percent_since_last": 4.9, + "cpu.freq.current": 2200.1639999999993, + "mem.total": 359053524992, + "mem.available": 349861822464, + "mem.used": 6035779584, + "mem.percent_used": 2.6, + "mem.read_bytes_since_boot": 31356966831104, + "mem.write_bytes_since_boot": 104324985856, + "net.bytes_sent_since_boot": 31338932, + "net.bytes_recv_since_boot": 31342034, + "gpu.count": 4, + "gpu.0.compute.util": 0.03, + "gpu.0.mem.util": 0.1278076171875, + "gpu.0.mem.total": 40960.0, + "gpu.0.mem.used": 5235.0, + "gpu.0.mem.free": 35092.0, + "gpu.0.temp.current": 34.0, + "gpu.1.compute.util": 0.08, + "gpu.1.mem.util": 0.1278076171875, + "gpu.1.mem.total": 40960.0, + "gpu.1.mem.used": 5235.0, + "gpu.1.mem.free": 35092.0, + "gpu.1.temp.current": 36.0, + "gpu.2.compute.util": 0.0, + "gpu.2.mem.util": 0.1278076171875, + "gpu.2.mem.total": 40960.0, + "gpu.2.mem.used": 5235.0, + "gpu.2.mem.free": 35092.0, + "gpu.2.temp.current": 33.0, + "gpu.3.compute.util": 0.08, + "gpu.3.mem.util": 0.1278076171875, + "gpu.3.mem.total": 40960.0, + "gpu.3.mem.used": 5235.0, + "gpu.3.mem.free": 35092.0, + "gpu.3.temp.current": 36.0, + "gpu.avg.compute.util": 0.0475, + "gpu.avg.mem.util": 0.1278076171875, + "gpu.avg.mem.total": 40960.0, + "gpu.avg.mem.used": 5235.0, + "gpu.avg.mem.free": 35092.0, + "gpu.avg.temp.current": 34.75, + "os_platform": "Linux-6.1.0-44-cloud-amd64-x86_64-with-glibc2.31", + "python_version": "3.11.10", + "python_compiler": "GCC 9.4.0", + "git_branch": "main", + "git_commit_hash": "b21be29be0a1573fb4f78f849aea019cdb520862", + "cpu_model_name": "Intel(R) Xeon(R) CPU @ 2.20GHz", + "cpu_count": 24, + "gpu_model_name": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_driver": "550.90.12", + "rng_seed": -858678099 +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_0/fastmri_pytorch/fastmri_pytorch_09-16-2026-06-26-24.log b/logs/self_tuning/ademamix_golden/study_0/fastmri_pytorch/fastmri_pytorch_09-16-2026-06-26-24.log new file mode 100644 index 00000000..b4ec04c6 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/fastmri_pytorch/fastmri_pytorch_09-16-2026-06-26-24.log @@ -0,0 +1,439 @@ +torchrun --redirects 1:0,2:0,3:0 --standalone --nnodes=1 --nproc_per_node=4 submission_runner.py --framework=pytorch --workload=fastmri --submission_path=submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py --data_dir=/data/fastmri --experiment_dir=/experiment_runs --experiment_name=submissions_a100/ademamix_golden/study_0 --overwrite=True --save_checkpoints=False --rng_seed=-1941863332 --torch_compile=true --tuning_ruleset=self 2>&1 | tee -a /logs/fastmri_pytorch_09-16-2026-06-26-24.log +W0916 06:26:51.477000 9 site-packages/torch/distributed/run.py:803] +W0916 06:26:51.477000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0916 06:26:51.477000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0916 06:26:51.477000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-16 06:27:06.937710: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 06:27:06.937700: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 06:27:06.937700: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 06:27:06.937700: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789540027.591522 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789540027.591524 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789540027.591527 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789540027.591526 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789540027.652978 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789540027.652987 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789540027.652990 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789540027.652996 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789540029.205945 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789540029.205937 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789540029.205938 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789540029.205938 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789540029.205980 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789540029.205980 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789540029.205980 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789540029.205982 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789540029.205983 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789540029.205984 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789540029.205984 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789540029.205985 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789540029.205986 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789540029.205986 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789540029.205987 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789540029.205987 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789540059.416510 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789540059.741696 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789540059.774098 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789540059.784117 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W916 06:27:58.965212225 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0916 06:28:01.072130 140634721277120 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/fastmri_pytorch. +I0916 06:28:01.072131 139790582932672 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/fastmri_pytorch. +I0916 06:28:01.072131 140405265020096 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/fastmri_pytorch. +I0916 06:28:01.072153 140525531075776 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/fastmri_pytorch. +I0916 06:28:01.287281 140525531075776 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/fastmri_pytorch/trial_1. +I0916 06:28:01.606939 140525531075776 submission_runner.py:242] Initializing dataset. +I0916 06:28:01.607111 140525531075776 submission_runner.py:251] Initializing model. +I0916 06:28:02.551614 140525531075776 submission_runner.py:290] Performing `torch.compile`. +I0916 06:28:06.727437 140525531075776 submission_runner.py:294] Initializing optimizer. +I0916 06:28:06.728096 140525531075776 submission_runner.py:299] Initializing metrics bundle. +I0916 06:28:06.728256 140525531075776 submission_runner.py:321] Initializing checkpoint and logger. +I0916 06:28:06.730078 140525531075776 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_0/fastmri_pytorch/trial_1/meta_data_0.json. +I0916 06:28:06.730144 140405265020096 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 06:28:06.730146 140634721277120 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 06:28:06.730160 139790582932672 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 06:28:06.730271 140525531075776 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 06:28:06.730307 140634721277120 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 06:28:06.730308 140405265020096 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 06:28:06.730326 140525531075776 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 06:28:06.730320 139790582932672 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 06:28:07.201400 140525531075776 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_0/fastmri_pytorch/trial_1/flags_0.json. +I0916 06:28:07.330509 140525531075776 submission_runner.py:359] Starting training loop. +[rank1]:W0916 06:28:07.492000 39 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank2]:W0916 06:28:07.492000 40 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank3]:W0916 06:28:07.492000 41 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +WARNING:tensorflow:AutoGraph could not transform and will run it as-is. +Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. +Cause: (, (leaf_jax_array := getattr(leaf, '__jax_array__', None))) +To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert +W0916 06:28:09.586269 140525531075776 ag_logging.py:142] AutoGraph could not transform and will run it as-is. +Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. +Cause: (, (leaf_jax_array := getattr(leaf, '__jax_array__', None))) +To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert +WARNING:tensorflow:AutoGraph could not transform and will run it as-is. +Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. +Cause: (, (aval := get_aval(a))) +To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert +W0916 06:28:13.239888 140525531075776 ag_logging.py:142] AutoGraph could not transform and will run it as-is. +Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. +Cause: (, (aval := get_aval(a))) +To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert +[rank0]:W0916 06:31:41.410000 38 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +I0916 06:32:57.396421 140492246923008 logging_writer.py:48] [0] global_step=0, grad_norm=0.5, loss=1.10816 +I0916 06:32:57.630623 140525531075776 submission.py:307] 0) loss = 1.108, grad_norm = 0.500 +I0916 06:32:58.267759 140525531075776 spec.py:333] Evaluating on the training split. +[rank1]:W0916 06:36:06.913000 39 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] torch._dynamo hit config.recompile_limit (8) +[rank1]:W0916 06:36:06.913000 39 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] function: 'forward' (/algorithmic-efficiency/algoperf/workloads/fastmri/fastmri_pytorch/models.py:139) +[rank1]:W0916 06:36:06.913000 39 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] last reason: 5/7: GLOBAL_STATE changed: grad_mode +[rank1]:W0916 06:36:06.913000 39 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To log all recompilation reasons, use TORCH_LOGS="recompiles". +[rank1]:W0916 06:36:06.913000 39 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To diagnose recompilation issues, see https://pytorch.org/docs/main/torch.compiler_troubleshooting.html +[rank0]:W0916 06:36:06.915000 38 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] torch._dynamo hit config.recompile_limit (8) +[rank0]:W0916 06:36:06.915000 38 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] function: 'forward' (/algorithmic-efficiency/algoperf/workloads/fastmri/fastmri_pytorch/models.py:139) +[rank0]:W0916 06:36:06.915000 38 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] last reason: 5/7: GLOBAL_STATE changed: grad_mode +[rank0]:W0916 06:36:06.915000 38 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To log all recompilation reasons, use TORCH_LOGS="recompiles". +[rank0]:W0916 06:36:06.915000 38 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To diagnose recompilation issues, see https://pytorch.org/docs/main/torch.compiler_troubleshooting.html +[rank2]:W0916 06:36:06.918000 40 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] torch._dynamo hit config.recompile_limit (8) +[rank2]:W0916 06:36:06.918000 40 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] function: 'forward' (/algorithmic-efficiency/algoperf/workloads/fastmri/fastmri_pytorch/models.py:139) +[rank2]:W0916 06:36:06.918000 40 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] last reason: 5/7: GLOBAL_STATE changed: grad_mode +[rank2]:W0916 06:36:06.918000 40 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To log all recompilation reasons, use TORCH_LOGS="recompiles". +[rank2]:W0916 06:36:06.918000 40 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To diagnose recompilation issues, see https://pytorch.org/docs/main/torch.compiler_troubleshooting.html +[rank3]:W0916 06:36:06.920000 41 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] torch._dynamo hit config.recompile_limit (8) +[rank3]:W0916 06:36:06.920000 41 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] function: 'forward' (/algorithmic-efficiency/algoperf/workloads/fastmri/fastmri_pytorch/models.py:139) +[rank3]:W0916 06:36:06.920000 41 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] last reason: 5/7: GLOBAL_STATE changed: grad_mode +[rank3]:W0916 06:36:06.920000 41 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To log all recompilation reasons, use TORCH_LOGS="recompiles". +[rank3]:W0916 06:36:06.920000 41 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To diagnose recompilation issues, see https://pytorch.org/docs/main/torch.compiler_troubleshooting.html +I0916 06:39:09.033555 140525531075776 spec.py:346] Evaluating on the validation split. +I0916 06:42:37.047281 140525531075776 spec.py:363] Evaluating on the test split. +I0916 06:46:07.680898 140525531075776 submission_runner.py:516] Time since start: 1080.35s, Step: 1, {'train/ssim': 0.2339261770248413, 'train/loss': 1.0799249921526228, 'validation/ssim': 0.2287525355178584, 'validation/loss': 1.0797452309457654, 'validation/num_examples': 3554, 'test/ssim': 0.25148652392837195, 'test/loss': 1.076364542245532, 'test/num_examples': 3581, 'score': 290.30148816108704, 'total_duration': 1080.350521326065, 'accumulated_submission_time': 290.30148816108704, 'accumulated_eval_time': 789.4131972789764, 'accumulated_logging_time': 0} +I0916 06:46:07.717592 140450009945856 logging_writer.py:48] [1] accumulated_eval_time=789.413, accumulated_logging_time=0, accumulated_submission_time=290.301, global_step=1, preemption_count=0, score=290.301, test/loss=1.07636, test/num_examples=3581, test/ssim=0.251487, total_duration=1080.35, train/loss=1.07992, train/ssim=0.233926, validation/loss=1.07975, validation/num_examples=3554, validation/ssim=0.228753 +I0916 06:46:08.440222 140450001553152 logging_writer.py:48] [1] global_step=1, grad_norm=0.5, loss=1.0786 +I0916 06:46:08.444185 140525531075776 submission.py:307] 1) loss = 1.079, grad_norm = 0.500 +I0916 06:46:08.538552 140450009945856 logging_writer.py:48] [2] global_step=2, grad_norm=0.5, loss=1.17688 +I0916 06:46:08.542256 140525531075776 submission.py:307] 2) loss = 1.177, grad_norm = 0.500 +I0916 06:46:08.633211 140450001553152 logging_writer.py:48] [3] global_step=3, grad_norm=0.5, loss=1.12993 +I0916 06:46:08.637100 140525531075776 submission.py:307] 3) loss = 1.130, grad_norm = 0.500 +I0916 06:46:08.731543 140450009945856 logging_writer.py:48] [4] global_step=4, grad_norm=0.5, loss=1.04344 +I0916 06:46:08.736630 140525531075776 submission.py:307] 4) loss = 1.043, grad_norm = 0.500 +I0916 06:46:08.826524 140450001553152 logging_writer.py:48] [5] global_step=5, grad_norm=0.5, loss=1.06463 +I0916 06:46:08.830217 140525531075776 submission.py:307] 5) loss = 1.065, grad_norm = 0.500 +I0916 06:46:08.920998 140450009945856 logging_writer.py:48] [6] global_step=6, grad_norm=0.5, loss=1.02322 +I0916 06:46:08.925101 140525531075776 submission.py:307] 6) loss = 1.023, grad_norm = 0.500 +I0916 06:46:09.017832 140450001553152 logging_writer.py:48] [7] global_step=7, grad_norm=0.5, loss=1.01679 +I0916 06:46:09.021719 140525531075776 submission.py:307] 7) loss = 1.017, grad_norm = 0.500 +I0916 06:46:09.102620 140450009945856 logging_writer.py:48] [8] global_step=8, grad_norm=0.5, loss=1.04147 +I0916 06:46:09.106543 140525531075776 submission.py:307] 8) loss = 1.041, grad_norm = 0.500 +I0916 06:46:09.199265 140450001553152 logging_writer.py:48] [9] global_step=9, grad_norm=0.5, loss=1.06425 +I0916 06:46:09.203480 140525531075776 submission.py:307] 9) loss = 1.064, grad_norm = 0.500 +I0916 06:46:09.284193 140450009945856 logging_writer.py:48] [10] global_step=10, grad_norm=0.5, loss=1.0823 +I0916 06:46:09.287798 140525531075776 submission.py:307] 10) loss = 1.082, grad_norm = 0.500 +I0916 06:46:09.373431 140450001553152 logging_writer.py:48] [11] global_step=11, grad_norm=0.5, loss=1.00249 +I0916 06:46:09.378360 140525531075776 submission.py:307] 11) loss = 1.002, grad_norm = 0.500 +I0916 06:46:09.460775 140450009945856 logging_writer.py:48] [12] global_step=12, grad_norm=0.5, loss=0.971304 +I0916 06:46:09.465886 140525531075776 submission.py:307] 12) loss = 0.971, grad_norm = 0.500 +I0916 06:46:09.545294 140450001553152 logging_writer.py:48] [13] global_step=13, grad_norm=0.5, loss=0.996273 +I0916 06:46:09.549036 140525531075776 submission.py:307] 13) loss = 0.996, grad_norm = 0.500 +I0916 06:46:09.632133 140450009945856 logging_writer.py:48] [14] global_step=14, grad_norm=0.5, loss=0.908595 +I0916 06:46:09.636375 140525531075776 submission.py:307] 14) loss = 0.909, grad_norm = 0.500 +I0916 06:46:09.717922 140450001553152 logging_writer.py:48] [15] global_step=15, grad_norm=0.5, loss=0.948871 +I0916 06:46:09.723777 140525531075776 submission.py:307] 15) loss = 0.949, grad_norm = 0.500 +I0916 06:46:09.803473 140450009945856 logging_writer.py:48] [16] global_step=16, grad_norm=0.5, loss=0.873616 +I0916 06:46:09.808469 140525531075776 submission.py:307] 16) loss = 0.874, grad_norm = 0.500 +I0916 06:46:09.893297 140450001553152 logging_writer.py:48] [17] global_step=17, grad_norm=0.5, loss=0.820508 +I0916 06:46:09.897098 140525531075776 submission.py:307] 17) loss = 0.821, grad_norm = 0.500 +I0916 06:46:09.974909 140450009945856 logging_writer.py:48] [18] global_step=18, grad_norm=0.5, loss=0.833346 +I0916 06:46:09.979517 140525531075776 submission.py:307] 18) loss = 0.833, grad_norm = 0.500 +I0916 06:46:10.065057 140450001553152 logging_writer.py:48] [19] global_step=19, grad_norm=0.5, loss=0.806311 +I0916 06:46:10.069306 140525531075776 submission.py:307] 19) loss = 0.806, grad_norm = 0.500 +I0916 06:46:10.147154 140450009945856 logging_writer.py:48] [20] global_step=20, grad_norm=0.5, loss=0.707922 +I0916 06:46:10.151087 140525531075776 submission.py:307] 20) loss = 0.708, grad_norm = 0.500 +I0916 06:46:10.239562 140450001553152 logging_writer.py:48] [21] global_step=21, grad_norm=0.5, loss=0.772723 +I0916 06:46:10.243444 140525531075776 submission.py:307] 21) loss = 0.773, grad_norm = 0.500 +I0916 06:46:10.324176 140450009945856 logging_writer.py:48] [22] global_step=22, grad_norm=0.5, loss=0.738027 +I0916 06:46:10.328062 140525531075776 submission.py:307] 22) loss = 0.738, grad_norm = 0.500 +I0916 06:46:10.413615 140450001553152 logging_writer.py:48] [23] global_step=23, grad_norm=0.5, loss=0.730492 +I0916 06:46:10.417997 140525531075776 submission.py:307] 23) loss = 0.730, grad_norm = 0.500 +I0916 06:46:10.500589 140450009945856 logging_writer.py:48] [24] global_step=24, grad_norm=0.5, loss=0.674925 +I0916 06:46:10.507331 140525531075776 submission.py:307] 24) loss = 0.675, grad_norm = 0.500 +I0916 06:46:10.584355 140450001553152 logging_writer.py:48] [25] global_step=25, grad_norm=0.5, loss=0.716002 +I0916 06:46:10.589589 140525531075776 submission.py:307] 25) loss = 0.716, grad_norm = 0.500 +I0916 06:46:10.673144 140450009945856 logging_writer.py:48] [26] global_step=26, grad_norm=0.5, loss=0.638053 +I0916 06:46:10.678913 140525531075776 submission.py:307] 26) loss = 0.638, grad_norm = 0.500 +I0916 06:46:10.762674 140450001553152 logging_writer.py:48] [27] global_step=27, grad_norm=0.5, loss=0.554656 +I0916 06:46:10.766727 140525531075776 submission.py:307] 27) loss = 0.555, grad_norm = 0.500 +I0916 06:46:10.846287 140450009945856 logging_writer.py:48] [28] global_step=28, grad_norm=0.5, loss=0.549669 +I0916 06:46:10.850759 140525531075776 submission.py:307] 28) loss = 0.550, grad_norm = 0.500 +I0916 06:46:10.933372 140450001553152 logging_writer.py:48] [29] global_step=29, grad_norm=0.5, loss=0.561574 +I0916 06:46:10.937592 140525531075776 submission.py:307] 29) loss = 0.562, grad_norm = 0.500 +I0916 06:46:11.020172 140450009945856 logging_writer.py:48] [30] global_step=30, grad_norm=0.5, loss=0.643226 +I0916 06:46:11.023864 140525531075776 submission.py:307] 30) loss = 0.643, grad_norm = 0.500 +I0916 06:46:11.103633 140450001553152 logging_writer.py:48] [31] global_step=31, grad_norm=0.5, loss=0.497802 +I0916 06:46:11.108226 140525531075776 submission.py:307] 31) loss = 0.498, grad_norm = 0.500 +I0916 06:46:11.182636 140450009945856 logging_writer.py:48] [32] global_step=32, grad_norm=0.5, loss=0.537321 +I0916 06:46:11.189140 140525531075776 submission.py:307] 32) loss = 0.537, grad_norm = 0.500 +I0916 06:46:11.270085 140450001553152 logging_writer.py:48] [33] global_step=33, grad_norm=0.5, loss=0.525104 +I0916 06:46:11.274191 140525531075776 submission.py:307] 33) loss = 0.525, grad_norm = 0.500 +I0916 06:46:11.356357 140450009945856 logging_writer.py:48] [34] global_step=34, grad_norm=0.5, loss=0.480024 +I0916 06:46:11.362106 140525531075776 submission.py:307] 34) loss = 0.480, grad_norm = 0.500 +I0916 06:46:11.447499 140450001553152 logging_writer.py:48] [35] global_step=35, grad_norm=0.5, loss=0.572586 +I0916 06:46:11.453181 140525531075776 submission.py:307] 35) loss = 0.573, grad_norm = 0.500 +I0916 06:46:11.522040 140450009945856 logging_writer.py:48] [36] global_step=36, grad_norm=0.5, loss=0.527868 +I0916 06:46:11.527314 140525531075776 submission.py:307] 36) loss = 0.528, grad_norm = 0.500 +I0916 06:46:11.613913 140450001553152 logging_writer.py:48] [37] global_step=37, grad_norm=0.5, loss=0.469718 +I0916 06:46:11.618791 140525531075776 submission.py:307] 37) loss = 0.470, grad_norm = 0.500 +I0916 06:46:11.696048 140450009945856 logging_writer.py:48] [38] global_step=38, grad_norm=0.5, loss=0.498505 +I0916 06:46:11.700284 140525531075776 submission.py:307] 38) loss = 0.499, grad_norm = 0.500 +I0916 06:46:11.786045 140450001553152 logging_writer.py:48] [39] global_step=39, grad_norm=0.499999, loss=0.424471 +I0916 06:46:11.790558 140525531075776 submission.py:307] 39) loss = 0.424, grad_norm = 0.500 +I0916 06:46:11.870719 140450009945856 logging_writer.py:48] [40] global_step=40, grad_norm=0.5, loss=0.506394 +I0916 06:46:11.874262 140525531075776 submission.py:307] 40) loss = 0.506, grad_norm = 0.500 +I0916 06:46:11.953790 140450001553152 logging_writer.py:48] [41] global_step=41, grad_norm=0.499999, loss=0.459761 +I0916 06:46:11.958085 140525531075776 submission.py:307] 41) loss = 0.460, grad_norm = 0.500 +I0916 06:46:12.045605 140450009945856 logging_writer.py:48] [42] global_step=42, grad_norm=0.499999, loss=0.454907 +I0916 06:46:12.051559 140525531075776 submission.py:307] 42) loss = 0.455, grad_norm = 0.500 +I0916 06:46:12.126114 140450001553152 logging_writer.py:48] [43] global_step=43, grad_norm=0.499999, loss=0.422366 +I0916 06:46:12.129864 140525531075776 submission.py:307] 43) loss = 0.422, grad_norm = 0.500 +I0916 06:46:12.215598 140450009945856 logging_writer.py:48] [44] global_step=44, grad_norm=0.499999, loss=0.493912 +I0916 06:46:12.219687 140525531075776 submission.py:307] 44) loss = 0.494, grad_norm = 0.500 +I0916 06:46:12.294485 140450001553152 logging_writer.py:48] [45] global_step=45, grad_norm=0.499999, loss=0.413667 +I0916 06:46:12.298588 140525531075776 submission.py:307] 45) loss = 0.414, grad_norm = 0.500 +I0916 06:46:12.377582 140450009945856 logging_writer.py:48] [46] global_step=46, grad_norm=0.499999, loss=0.404987 +I0916 06:46:12.382423 140525531075776 submission.py:307] 46) loss = 0.405, grad_norm = 0.500 +I0916 06:46:12.464613 140450001553152 logging_writer.py:48] [47] global_step=47, grad_norm=0.499999, loss=0.375358 +I0916 06:46:12.468562 140525531075776 submission.py:307] 47) loss = 0.375, grad_norm = 0.500 +I0916 06:46:12.543963 140450009945856 logging_writer.py:48] [48] global_step=48, grad_norm=0.499999, loss=0.376415 +I0916 06:46:12.549950 140525531075776 submission.py:307] 48) loss = 0.376, grad_norm = 0.500 +I0916 06:46:12.629243 140450001553152 logging_writer.py:48] [49] global_step=49, grad_norm=0.499999, loss=0.347893 +I0916 06:46:12.634332 140525531075776 submission.py:307] 49) loss = 0.348, grad_norm = 0.500 +I0916 06:46:12.971113 140450009945856 logging_writer.py:48] [50] global_step=50, grad_norm=0.499999, loss=0.343566 +I0916 06:46:12.976678 140525531075776 submission.py:307] 50) loss = 0.344, grad_norm = 0.500 +I0916 06:46:13.368869 140450001553152 logging_writer.py:48] [51] global_step=51, grad_norm=0.499999, loss=0.420119 +I0916 06:46:13.372614 140525531075776 submission.py:307] 51) loss = 0.420, grad_norm = 0.500 +I0916 06:46:13.592839 140450009945856 logging_writer.py:48] [52] global_step=52, grad_norm=0.473414, loss=0.389686 +I0916 06:46:13.597170 140525531075776 submission.py:307] 52) loss = 0.390, grad_norm = 0.473 +I0916 06:46:13.716683 140450001553152 logging_writer.py:48] [53] global_step=53, grad_norm=0.478594, loss=0.430258 +I0916 06:46:13.720841 140525531075776 submission.py:307] 53) loss = 0.430, grad_norm = 0.479 +I0916 06:46:13.831526 140450009945856 logging_writer.py:48] [54] global_step=54, grad_norm=0.469902, loss=0.478319 +I0916 06:46:13.839992 140525531075776 submission.py:307] 54) loss = 0.478, grad_norm = 0.470 +I0916 06:46:14.013962 140450001553152 logging_writer.py:48] [55] global_step=55, grad_norm=0.356532, loss=0.327923 +I0916 06:46:14.017769 140525531075776 submission.py:307] 55) loss = 0.328, grad_norm = 0.357 +I0916 06:46:14.315510 140450009945856 logging_writer.py:48] [56] global_step=56, grad_norm=0.345777, loss=0.363484 +I0916 06:46:14.319427 140525531075776 submission.py:307] 56) loss = 0.363, grad_norm = 0.346 +I0916 06:46:14.504030 140450001553152 logging_writer.py:48] [57] global_step=57, grad_norm=0.331312, loss=0.35091 +I0916 06:46:14.509005 140525531075776 submission.py:307] 57) loss = 0.351, grad_norm = 0.331 +I0916 06:46:14.754454 140450009945856 logging_writer.py:48] [58] global_step=58, grad_norm=0.337904, loss=0.370225 +I0916 06:46:14.760209 140525531075776 submission.py:307] 58) loss = 0.370, grad_norm = 0.338 +I0916 06:46:14.902322 140450001553152 logging_writer.py:48] [59] global_step=59, grad_norm=0.310824, loss=0.496749 +I0916 06:46:14.906885 140525531075776 submission.py:307] 59) loss = 0.497, grad_norm = 0.311 +I0916 06:46:15.011238 140450009945856 logging_writer.py:48] [60] global_step=60, grad_norm=0.240517, loss=0.290525 +I0916 06:46:15.016407 140525531075776 submission.py:307] 60) loss = 0.291, grad_norm = 0.241 +I0916 06:46:15.108644 140450001553152 logging_writer.py:48] [61] global_step=61, grad_norm=0.25224, loss=0.420208 +I0916 06:46:15.112976 140525531075776 submission.py:307] 61) loss = 0.420, grad_norm = 0.252 +I0916 06:46:15.215062 140450009945856 logging_writer.py:48] [62] global_step=62, grad_norm=0.235797, loss=0.369462 +I0916 06:46:15.220059 140525531075776 submission.py:307] 62) loss = 0.369, grad_norm = 0.236 +I0916 06:46:15.322100 140450001553152 logging_writer.py:48] [63] global_step=63, grad_norm=0.499999, loss=0.291197 +I0916 06:46:15.329928 140525531075776 submission.py:307] 63) loss = 0.291, grad_norm = 0.500 +I0916 06:46:15.416347 140450009945856 logging_writer.py:48] [64] global_step=64, grad_norm=0.320932, loss=0.392183 +I0916 06:46:15.420338 140525531075776 submission.py:307] 64) loss = 0.392, grad_norm = 0.321 +I0916 06:46:15.505845 140450001553152 logging_writer.py:48] [65] global_step=65, grad_norm=0.350159, loss=0.389636 +I0916 06:46:15.511621 140525531075776 submission.py:307] 65) loss = 0.390, grad_norm = 0.350 +I0916 06:46:15.589250 140450009945856 logging_writer.py:48] [66] global_step=66, grad_norm=0.288735, loss=0.314207 +I0916 06:46:15.593448 140525531075776 submission.py:307] 66) loss = 0.314, grad_norm = 0.289 +I0916 06:46:15.668750 140450001553152 logging_writer.py:48] [67] global_step=67, grad_norm=0.255971, loss=0.3353 +I0916 06:46:15.674322 140525531075776 submission.py:307] 67) loss = 0.335, grad_norm = 0.256 +I0916 06:46:15.791290 140450009945856 logging_writer.py:48] [68] global_step=68, grad_norm=0.266288, loss=0.330366 +I0916 06:46:15.795655 140525531075776 submission.py:307] 68) loss = 0.330, grad_norm = 0.266 +I0916 06:46:15.964394 140450001553152 logging_writer.py:48] [69] global_step=69, grad_norm=0.264428, loss=0.320592 +I0916 06:46:15.968845 140525531075776 submission.py:307] 69) loss = 0.321, grad_norm = 0.264 +I0916 06:46:16.119856 140450009945856 logging_writer.py:48] [70] global_step=70, grad_norm=0.181992, loss=0.495692 +I0916 06:46:16.123859 140525531075776 submission.py:307] 70) loss = 0.496, grad_norm = 0.182 +I0916 06:46:16.325302 140450001553152 logging_writer.py:48] [71] global_step=71, grad_norm=0.231637, loss=0.314661 +I0916 06:46:16.329683 140525531075776 submission.py:307] 71) loss = 0.315, grad_norm = 0.232 +I0916 06:46:16.475758 140450009945856 logging_writer.py:48] [72] global_step=72, grad_norm=0.144998, loss=0.332344 +I0916 06:46:16.480880 140525531075776 submission.py:307] 72) loss = 0.332, grad_norm = 0.145 +I0916 06:46:16.632530 140450001553152 logging_writer.py:48] [73] global_step=73, grad_norm=0.214117, loss=0.350018 +I0916 06:46:16.638595 140525531075776 submission.py:307] 73) loss = 0.350, grad_norm = 0.214 +I0916 06:46:16.760870 140450009945856 logging_writer.py:48] [74] global_step=74, grad_norm=0.206912, loss=0.368474 +I0916 06:46:16.765424 140525531075776 submission.py:307] 74) loss = 0.368, grad_norm = 0.207 +I0916 06:46:16.916732 140450001553152 logging_writer.py:48] [75] global_step=75, grad_norm=0.218166, loss=0.355828 +I0916 06:46:16.920848 140525531075776 submission.py:307] 75) loss = 0.356, grad_norm = 0.218 +I0916 06:46:17.084865 140450009945856 logging_writer.py:48] [76] global_step=76, grad_norm=0.320186, loss=0.341377 +I0916 06:46:17.089233 140525531075776 submission.py:307] 76) loss = 0.341, grad_norm = 0.320 +I0916 06:46:17.231477 140450001553152 logging_writer.py:48] [77] global_step=77, grad_norm=0.298341, loss=0.312377 +I0916 06:46:17.236660 140525531075776 submission.py:307] 77) loss = 0.312, grad_norm = 0.298 +I0916 06:46:17.343816 140450009945856 logging_writer.py:48] [78] global_step=78, grad_norm=0.166851, loss=0.324523 +I0916 06:46:17.348233 140525531075776 submission.py:307] 78) loss = 0.325, grad_norm = 0.167 +I0916 06:46:17.460103 140450001553152 logging_writer.py:48] [79] global_step=79, grad_norm=0.298017, loss=0.443312 +I0916 06:46:17.464163 140525531075776 submission.py:307] 79) loss = 0.443, grad_norm = 0.298 +I0916 06:46:17.692527 140450009945856 logging_writer.py:48] [80] global_step=80, grad_norm=0.230228, loss=0.324792 +I0916 06:46:17.697404 140525531075776 submission.py:307] 80) loss = 0.325, grad_norm = 0.230 +I0916 06:46:17.899176 140450001553152 logging_writer.py:48] [81] global_step=81, grad_norm=0.248092, loss=0.283758 +I0916 06:46:17.904194 140525531075776 submission.py:307] 81) loss = 0.284, grad_norm = 0.248 +I0916 06:46:18.062992 140450009945856 logging_writer.py:48] [82] global_step=82, grad_norm=0.218687, loss=0.493305 +I0916 06:46:18.067001 140525531075776 submission.py:307] 82) loss = 0.493, grad_norm = 0.219 +I0916 06:46:18.304527 140450001553152 logging_writer.py:48] [83] global_step=83, grad_norm=0.18347, loss=0.405487 +I0916 06:46:18.309804 140525531075776 submission.py:307] 83) loss = 0.405, grad_norm = 0.183 +I0916 06:46:18.693203 140450009945856 logging_writer.py:48] [84] global_step=84, grad_norm=0.30866, loss=0.323085 +I0916 06:46:18.697350 140525531075776 submission.py:307] 84) loss = 0.323, grad_norm = 0.309 +I0916 06:46:18.916904 140450001553152 logging_writer.py:48] [85] global_step=85, grad_norm=0.213958, loss=0.339333 +I0916 06:46:18.921132 140525531075776 submission.py:307] 85) loss = 0.339, grad_norm = 0.214 +I0916 06:46:19.155865 140450009945856 logging_writer.py:48] [86] global_step=86, grad_norm=0.272619, loss=0.345292 +I0916 06:46:19.161127 140525531075776 submission.py:307] 86) loss = 0.345, grad_norm = 0.273 +I0916 06:46:19.304324 140450001553152 logging_writer.py:48] [87] global_step=87, grad_norm=0.292895, loss=0.378356 +I0916 06:46:19.308144 140525531075776 submission.py:307] 87) loss = 0.378, grad_norm = 0.293 +I0916 06:46:19.423455 140450009945856 logging_writer.py:48] [88] global_step=88, grad_norm=0.252451, loss=0.391686 +I0916 06:46:19.428899 140525531075776 submission.py:307] 88) loss = 0.392, grad_norm = 0.252 +I0916 06:46:19.594177 140450001553152 logging_writer.py:48] [89] global_step=89, grad_norm=0.25316, loss=0.372057 +I0916 06:46:19.598295 140525531075776 submission.py:307] 89) loss = 0.372, grad_norm = 0.253 +I0916 06:46:19.790220 140450009945856 logging_writer.py:48] [90] global_step=90, grad_norm=0.300363, loss=0.309762 +I0916 06:46:19.797403 140525531075776 submission.py:307] 90) loss = 0.310, grad_norm = 0.300 +I0916 06:46:19.949862 140450001553152 logging_writer.py:48] [91] global_step=91, grad_norm=0.170716, loss=0.355084 +I0916 06:46:19.954047 140525531075776 submission.py:307] 91) loss = 0.355, grad_norm = 0.171 +I0916 06:46:20.145418 140450009945856 logging_writer.py:48] [92] global_step=92, grad_norm=0.303096, loss=0.409551 +I0916 06:46:20.149988 140525531075776 submission.py:307] 92) loss = 0.410, grad_norm = 0.303 +I0916 06:46:20.325823 140450001553152 logging_writer.py:48] [93] global_step=93, grad_norm=0.206015, loss=0.437601 +I0916 06:46:20.330414 140525531075776 submission.py:307] 93) loss = 0.438, grad_norm = 0.206 +I0916 06:46:20.408119 140450009945856 logging_writer.py:48] [94] global_step=94, grad_norm=0.311011, loss=0.40485 +I0916 06:46:20.413227 140525531075776 submission.py:307] 94) loss = 0.405, grad_norm = 0.311 +I0916 06:46:20.528632 140450001553152 logging_writer.py:48] [95] global_step=95, grad_norm=0.208429, loss=0.338736 +I0916 06:46:20.533643 140525531075776 submission.py:307] 95) loss = 0.339, grad_norm = 0.208 +I0916 06:46:20.614666 140450009945856 logging_writer.py:48] [96] global_step=96, grad_norm=0.200315, loss=0.375218 +I0916 06:46:20.618626 140525531075776 submission.py:307] 96) loss = 0.375, grad_norm = 0.200 +I0916 06:46:20.720124 140450001553152 logging_writer.py:48] [97] global_step=97, grad_norm=0.346072, loss=0.321338 +I0916 06:46:20.724069 140525531075776 submission.py:307] 97) loss = 0.321, grad_norm = 0.346 +I0916 06:46:20.835281 140450009945856 logging_writer.py:48] [98] global_step=98, grad_norm=0.191307, loss=0.395343 +I0916 06:46:20.840825 140525531075776 submission.py:307] 98) loss = 0.395, grad_norm = 0.191 +I0916 06:46:20.941935 140450001553152 logging_writer.py:48] [99] global_step=99, grad_norm=0.240288, loss=0.314272 +I0916 06:46:20.946839 140525531075776 submission.py:307] 99) loss = 0.314, grad_norm = 0.240 +I0916 06:46:21.071903 140450009945856 logging_writer.py:48] [100] global_step=100, grad_norm=0.233247, loss=0.257968 +I0916 06:46:21.076383 140525531075776 submission.py:307] 100) loss = 0.258, grad_norm = 0.233 +I0916 06:47:58.463034 140525531075776 spec.py:333] Evaluating on the training split. +I0916 06:48:00.460459 140525531075776 spec.py:346] Evaluating on the validation split. +I0916 06:48:03.925375 140525531075776 spec.py:363] Evaluating on the test split. +I0916 06:48:06.322606 140525531075776 submission_runner.py:516] Time since start: 1198.99s, Step: 265, {'train/ssim': 0.7135448455810547, 'train/loss': 0.2963003771645682, 'validation/ssim': 0.6889925276756823, 'validation/loss': 0.3176315556635129, 'validation/num_examples': 3554, 'test/ssim': 0.7065782842606814, 'test/loss': 0.3197439748193591, 'test/num_examples': 3581, 'score': 399.7377872467041, 'total_duration': 1198.9923601150513, 'accumulated_submission_time': 399.7377872467041, 'accumulated_eval_time': 797.2729263305664, 'accumulated_logging_time': 0.045484304428100586} +I0916 06:48:06.344792 140450001553152 logging_writer.py:48] [265] accumulated_eval_time=797.273, accumulated_logging_time=0.0454843, accumulated_submission_time=399.738, global_step=265, preemption_count=0, score=399.738, test/loss=0.319744, test/num_examples=3581, test/ssim=0.706578, total_duration=1198.99, train/loss=0.2963, train/ssim=0.713545, validation/loss=0.317632, validation/num_examples=3554, validation/ssim=0.688993 +I0916 06:49:59.104302 140525531075776 spec.py:333] Evaluating on the training split. +I0916 06:50:01.118544 140525531075776 spec.py:346] Evaluating on the validation split. +I0916 06:50:03.590114 140525531075776 spec.py:363] Evaluating on the test split. +I0916 06:50:05.897634 140525531075776 submission_runner.py:516] Time since start: 1318.57s, Step: 328, {'train/ssim': 0.721343994140625, 'train/loss': 0.2902473381587437, 'validation/ssim': 0.6961671295767093, 'validation/loss': 0.3113961120502427, 'validation/num_examples': 3554, 'test/ssim': 0.7138226681400097, 'test/loss': 0.3135263655512601, 'test/num_examples': 3581, 'score': 511.3699195384979, 'total_duration': 1318.5673644542694, 'accumulated_submission_time': 511.3699195384979, 'accumulated_eval_time': 804.0663478374481, 'accumulated_logging_time': 0.0765845775604248} +I0916 06:50:05.919518 140450009945856 logging_writer.py:48] [328] accumulated_eval_time=804.066, accumulated_logging_time=0.0765846, accumulated_submission_time=511.37, global_step=328, preemption_count=0, score=511.37, test/loss=0.313526, test/num_examples=3581, test/ssim=0.713823, total_duration=1318.57, train/loss=0.290247, train/ssim=0.721344, validation/loss=0.311396, validation/num_examples=3554, validation/ssim=0.696167 +I0916 06:51:57.564507 140525531075776 spec.py:333] Evaluating on the training split. +I0916 06:51:59.562755 140525531075776 spec.py:346] Evaluating on the validation split. +I0916 06:52:02.043981 140525531075776 spec.py:363] Evaluating on the test split. +I0916 06:52:05.296985 140525531075776 submission_runner.py:516] Time since start: 1437.97s, Step: 395, {'train/ssim': 0.7220333644321987, 'train/loss': 0.28904570852007183, 'validation/ssim': 0.6983517553944499, 'validation/loss': 0.3094093610412563, 'validation/num_examples': 3554, 'test/ssim': 0.7155941705136484, 'test/loss': 0.31169029987913643, 'test/num_examples': 3581, 'score': 621.8826193809509, 'total_duration': 1437.9666678905487, 'accumulated_submission_time': 621.8826193809509, 'accumulated_eval_time': 811.7989797592163, 'accumulated_logging_time': 0.10672640800476074} +I0916 06:52:05.319024 140450001553152 logging_writer.py:48] [395] accumulated_eval_time=811.799, accumulated_logging_time=0.106726, accumulated_submission_time=621.883, global_step=395, preemption_count=0, score=621.883, test/loss=0.31169, test/num_examples=3581, test/ssim=0.715594, total_duration=1437.97, train/loss=0.289046, train/ssim=0.722033, validation/loss=0.309409, validation/num_examples=3554, validation/ssim=0.698352 +I0916 06:53:56.581887 140525531075776 spec.py:333] Evaluating on the training split. +I0916 06:53:58.592483 140525531075776 spec.py:346] Evaluating on the validation split. +I0916 06:54:01.037463 140525531075776 spec.py:363] Evaluating on the test split. +I0916 06:54:04.345326 140525531075776 submission_runner.py:516] Time since start: 1557.02s, Step: 460, {'train/ssim': 0.7258890015738351, 'train/loss': 0.2857875483376639, 'validation/ssim': 0.700805045811058, 'validation/loss': 0.3066876808041995, 'validation/num_examples': 3554, 'test/ssim': 0.7179462653370218, 'test/loss': 0.3090552037336812, 'test/num_examples': 3581, 'score': 732.0183448791504, 'total_duration': 1557.0150799751282, 'accumulated_submission_time': 732.0183448791504, 'accumulated_eval_time': 819.5625236034393, 'accumulated_logging_time': 0.13713788986206055} +I0916 06:54:04.368046 140450009945856 logging_writer.py:48] [460] accumulated_eval_time=819.563, accumulated_logging_time=0.137138, accumulated_submission_time=732.018, global_step=460, preemption_count=0, score=732.018, test/loss=0.309055, test/num_examples=3581, test/ssim=0.717946, total_duration=1557.02, train/loss=0.285788, train/ssim=0.725889, validation/loss=0.306688, validation/num_examples=3554, validation/ssim=0.700805 +I0916 06:55:11.465961 140450001553152 logging_writer.py:48] [500] global_step=500, grad_norm=0.266472, loss=0.251947 +I0916 06:55:11.469232 140525531075776 submission.py:307] 500) loss = 0.252, grad_norm = 0.266 +I0916 06:55:55.377687 140525531075776 spec.py:333] Evaluating on the training split. +I0916 06:55:57.374502 140525531075776 spec.py:346] Evaluating on the validation split. +I0916 06:56:00.792266 140525531075776 spec.py:363] Evaluating on the test split. +I0916 06:56:03.124242 140525531075776 submission_runner.py:516] Time since start: 1675.79s, Step: 526, {'train/ssim': 0.7302299908229283, 'train/loss': 0.2826956680842808, 'validation/ssim': 0.7050498228228756, 'validation/loss': 0.30353507931204277, 'validation/num_examples': 3554, 'test/ssim': 0.7223764529810109, 'test/loss': 0.30563876895856606, 'test/num_examples': 3581, 'score': 841.8979823589325, 'total_duration': 1675.7940037250519, 'accumulated_submission_time': 841.8979823589325, 'accumulated_eval_time': 827.3092002868652, 'accumulated_logging_time': 0.16816020011901855} +I0916 06:56:03.146430 140450009945856 logging_writer.py:48] [526] accumulated_eval_time=827.309, accumulated_logging_time=0.16816, accumulated_submission_time=841.898, global_step=526, preemption_count=0, score=841.898, test/loss=0.305639, test/num_examples=3581, test/ssim=0.722376, total_duration=1675.79, train/loss=0.282696, train/ssim=0.73023, validation/loss=0.303535, validation/num_examples=3554, validation/ssim=0.70505 +I0916 06:57:54.440171 140525531075776 spec.py:333] Evaluating on the training split. +I0916 06:57:56.436797 140525531075776 spec.py:346] Evaluating on the validation split. +I0916 06:57:58.977845 140525531075776 spec.py:363] Evaluating on the test split. +I0916 06:58:02.207857 140525531075776 submission_runner.py:516] Time since start: 1794.88s, Step: 599, {'train/ssim': 0.7273873601640973, 'train/loss': 0.2828923463821411, 'validation/ssim': 0.7033286108082443, 'validation/loss': 0.3033684948957161, 'validation/num_examples': 3554, 'test/ssim': 0.7207503032497906, 'test/loss': 0.3053292469151424, 'test/num_examples': 3581, 'score': 952.0686573982239, 'total_duration': 1794.8776230812073, 'accumulated_submission_time': 952.0686573982239, 'accumulated_eval_time': 835.0769908428192, 'accumulated_logging_time': 0.19854354858398438} +I0916 06:58:02.232717 140450001553152 logging_writer.py:48] [599] accumulated_eval_time=835.077, accumulated_logging_time=0.198544, accumulated_submission_time=952.069, global_step=599, preemption_count=0, score=952.069, test/loss=0.305329, test/num_examples=3581, test/ssim=0.72075, total_duration=1794.88, train/loss=0.282892, train/ssim=0.727387, validation/loss=0.303368, validation/num_examples=3554, validation/ssim=0.703329 +I0916 06:59:53.860934 140525531075776 spec.py:333] Evaluating on the training split. +I0916 06:59:55.867671 140525531075776 spec.py:346] Evaluating on the validation split. +I0916 06:59:59.543565 140525531075776 spec.py:363] Evaluating on the test split. +I0916 07:00:02.090801 140525531075776 submission_runner.py:516] Time since start: 1914.76s, Step: 667, {'train/ssim': 0.733936037336077, 'train/loss': 0.27998731817517963, 'validation/ssim': 0.7085141604310284, 'validation/loss': 0.30102236806239446, 'validation/num_examples': 3554, 'test/ssim': 0.7256016181321558, 'test/loss': 0.30322282669165385, 'test/num_examples': 3581, 'score': 1062.5406551361084, 'total_duration': 1914.760534286499, 'accumulated_submission_time': 1062.5406551361084, 'accumulated_eval_time': 843.3068838119507, 'accumulated_logging_time': 0.23186635971069336} +I0916 07:00:02.115949 140450009945856 logging_writer.py:48] [667] accumulated_eval_time=843.307, accumulated_logging_time=0.231866, accumulated_submission_time=1062.54, global_step=667, preemption_count=0, score=1062.54, test/loss=0.303223, test/num_examples=3581, test/ssim=0.725602, total_duration=1914.76, train/loss=0.279987, train/ssim=0.733936, validation/loss=0.301022, validation/num_examples=3554, validation/ssim=0.708514 +I0916 07:01:53.790108 140525531075776 spec.py:333] Evaluating on the training split. +I0916 07:01:55.773663 140525531075776 spec.py:346] Evaluating on the validation split. +I0916 07:01:59.299554 140525531075776 spec.py:363] Evaluating on the test split. +I0916 07:02:01.588883 140525531075776 submission_runner.py:516] Time since start: 2034.26s, Step: 731, {'train/ssim': 0.7337468692234584, 'train/loss': 0.2794912202017648, 'validation/ssim': 0.7086373298572032, 'validation/loss': 0.30034239451806066, 'validation/num_examples': 3554, 'test/ssim': 0.7259842255567579, 'test/loss': 0.3023471315624127, 'test/num_examples': 3581, 'score': 1173.0753722190857, 'total_duration': 2034.258635044098, 'accumulated_submission_time': 1173.0753722190857, 'accumulated_eval_time': 851.1057827472687, 'accumulated_logging_time': 0.26522254943847656} +I0916 07:02:01.611593 140450001553152 logging_writer.py:48] [731] accumulated_eval_time=851.106, accumulated_logging_time=0.265223, accumulated_submission_time=1173.08, global_step=731, preemption_count=0, score=1173.08, test/loss=0.302347, test/num_examples=3581, test/ssim=0.725984, total_duration=2034.26, train/loss=0.279491, train/ssim=0.733747, validation/loss=0.300342, validation/num_examples=3554, validation/ssim=0.708637 +I0916 07:03:53.973680 140525531075776 spec.py:333] Evaluating on the training split. +I0916 07:03:55.965289 140525531075776 spec.py:346] Evaluating on the validation split. +I0916 07:03:59.417507 140525531075776 spec.py:363] Evaluating on the test split. +I0916 07:04:01.965991 140525531075776 submission_runner.py:516] Time since start: 2154.64s, Step: 800, {'train/ssim': 0.7359416144234794, 'train/loss': 0.2779566901070731, 'validation/ssim': 0.7105753420441756, 'validation/loss': 0.29897430711873946, 'validation/num_examples': 3554, 'test/ssim': 0.7276828470792027, 'test/loss': 0.30104437782524085, 'test/num_examples': 3581, 'score': 1284.2708780765533, 'total_duration': 2154.6357412338257, 'accumulated_submission_time': 1284.2708780765533, 'accumulated_eval_time': 859.098197221756, 'accumulated_logging_time': 0.296309232711792} +I0916 07:04:01.991872 140450009945856 logging_writer.py:48] [800] accumulated_eval_time=859.098, accumulated_logging_time=0.296309, accumulated_submission_time=1284.27, global_step=800, preemption_count=0, score=1284.27, test/loss=0.301044, test/num_examples=3581, test/ssim=0.727683, total_duration=2154.64, train/loss=0.277957, train/ssim=0.735942, validation/loss=0.298974, validation/num_examples=3554, validation/ssim=0.710575 +I0916 07:05:53.696561 140525531075776 spec.py:333] Evaluating on the training split. +I0916 07:05:55.677920 140525531075776 spec.py:346] Evaluating on the validation split. +I0916 07:05:58.147402 140525531075776 spec.py:363] Evaluating on the test split. +I0916 07:06:01.228832 140525531075776 submission_runner.py:516] Time since start: 2273.90s, Step: 864, {'train/ssim': 0.7358097348894391, 'train/loss': 0.2772273676736014, 'validation/ssim': 0.7106857342738815, 'validation/loss': 0.29819840155898286, 'validation/num_examples': 3554, 'test/ssim': 0.7278560839761938, 'test/loss': 0.3002523013713872, 'test/num_examples': 3581, 'score': 1394.8430495262146, 'total_duration': 2273.898585319519, 'accumulated_submission_time': 1394.8430495262146, 'accumulated_eval_time': 866.6306080818176, 'accumulated_logging_time': 0.33432435989379883} +I0916 07:06:01.252709 140450001553152 logging_writer.py:48] [864] accumulated_eval_time=866.631, accumulated_logging_time=0.334324, accumulated_submission_time=1394.84, global_step=864, preemption_count=0, score=1394.84, test/loss=0.300252, test/num_examples=3581, test/ssim=0.727856, total_duration=2273.9, train/loss=0.277227, train/ssim=0.73581, validation/loss=0.298198, validation/num_examples=3554, validation/ssim=0.710686 +I0916 07:07:51.978590 140525531075776 spec.py:333] Evaluating on the training split. +I0916 07:07:53.963863 140525531075776 spec.py:346] Evaluating on the validation split. +I0916 07:07:57.389284 140525531075776 spec.py:363] Evaluating on the test split. +I0916 07:08:00.534595 140525531075776 submission_runner.py:516] Time since start: 2393.20s, Step: 935, {'train/ssim': 0.735457352229527, 'train/loss': 0.2781445639474051, 'validation/ssim': 0.7099429395135762, 'validation/loss': 0.2993945120155107, 'validation/num_examples': 3554, 'test/ssim': 0.7269426530647863, 'test/loss': 0.30146649362520944, 'test/num_examples': 3581, 'score': 1504.4105689525604, 'total_duration': 2393.2043352127075, 'accumulated_submission_time': 1504.4105689525604, 'accumulated_eval_time': 875.186728477478, 'accumulated_logging_time': 0.3900284767150879} +I0916 07:08:00.558963 140450009945856 logging_writer.py:48] [935] accumulated_eval_time=875.187, accumulated_logging_time=0.390028, accumulated_submission_time=1504.41, global_step=935, preemption_count=0, score=1504.41, test/loss=0.301466, test/num_examples=3581, test/ssim=0.726943, total_duration=2393.2, train/loss=0.278145, train/ssim=0.735457, validation/loss=0.299395, validation/num_examples=3554, validation/ssim=0.709943 +I0916 07:08:45.972252 140450001553152 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.119466, loss=0.258767 +I0916 07:08:45.978857 140525531075776 submission.py:307] 1000) loss = 0.259, grad_norm = 0.119 +I0916 07:09:23.878490 140450009945856 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.234117, loss=0.261701 +I0916 07:09:23.881598 140525531075776 submission.py:307] 1500) loss = 0.262, grad_norm = 0.234 +I0916 07:09:48.029749 140450001553152 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.201167, loss=0.256685 +I0916 07:09:48.032912 140525531075776 submission.py:307] 2000) loss = 0.257, grad_norm = 0.201 +I0916 07:09:51.099646 140525531075776 spec.py:333] Evaluating on the training split. +I0916 07:09:53.051377 140525531075776 spec.py:346] Evaluating on the validation split. +I0916 07:09:56.373753 140525531075776 spec.py:363] Evaluating on the test split. +I0916 07:09:58.465353 140525531075776 submission_runner.py:516] Time since start: 2511.14s, Step: 2053, {'train/ssim': 0.7436562946864537, 'train/loss': 0.2696697541645595, 'validation/ssim': 0.7178362243686691, 'validation/loss': 0.291437341178074, 'validation/num_examples': 3554, 'test/ssim': 0.7350217238114354, 'test/loss': 0.2930827413781067, 'test/num_examples': 3581, 'score': 1613.449716091156, 'total_duration': 2511.1351025104523, 'accumulated_submission_time': 1613.449716091156, 'accumulated_eval_time': 882.5525469779968, 'accumulated_logging_time': 0.42283177375793457} +I0916 07:09:58.486963 140450009945856 logging_writer.py:48] [2053] accumulated_eval_time=882.553, accumulated_logging_time=0.422832, accumulated_submission_time=1613.45, global_step=2053, preemption_count=0, score=1613.45, test/loss=0.293083, test/num_examples=3581, test/ssim=0.735022, total_duration=2511.14, train/loss=0.26967, train/ssim=0.743656, validation/loss=0.291437, validation/num_examples=3554, validation/ssim=0.717836 +I0916 07:10:20.710112 140450001553152 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.0770029, loss=0.313379 +I0916 07:10:20.713200 140525531075776 submission.py:307] 2500) loss = 0.313, grad_norm = 0.077 +I0916 07:10:44.925415 140450009945856 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.182453, loss=0.268634 +I0916 07:10:44.928658 140525531075776 submission.py:307] 3000) loss = 0.269, grad_norm = 0.182 +I0916 07:11:09.139747 140450001553152 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.313898, loss=0.279093 +I0916 07:11:09.142916 140525531075776 submission.py:307] 3500) loss = 0.279, grad_norm = 0.314 +I0916 07:11:33.401885 140450009945856 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.075241, loss=0.270752 +I0916 07:11:33.405152 140525531075776 submission.py:307] 4000) loss = 0.271, grad_norm = 0.075 +I0916 07:11:49.036527 140525531075776 spec.py:333] Evaluating on the training split. +I0916 07:11:50.992079 140525531075776 spec.py:346] Evaluating on the validation split. +I0916 07:11:55.009261 140525531075776 spec.py:363] Evaluating on the test split. +I0916 07:11:57.268773 140525531075776 submission_runner.py:516] Time since start: 2629.94s, Step: 4314, {'train/ssim': 0.7471103668212891, 'train/loss': 0.2660341433116368, 'validation/ssim': 0.7213369701172974, 'validation/loss': 0.28785474580798043, 'validation/num_examples': 3554, 'test/ssim': 0.7384776669619171, 'test/loss': 0.28935490970137534, 'test/num_examples': 3581, 'score': 1722.1367993354797, 'total_duration': 2629.9385311603546, 'accumulated_submission_time': 1722.1367993354797, 'accumulated_eval_time': 890.7848875522614, 'accumulated_logging_time': 0.4526228904724121} +I0916 07:11:57.291848 140450001553152 logging_writer.py:48] [4314] accumulated_eval_time=890.785, accumulated_logging_time=0.452623, accumulated_submission_time=1722.14, global_step=4314, preemption_count=0, score=1722.14, test/loss=0.289355, test/num_examples=3581, test/ssim=0.738478, total_duration=2629.94, train/loss=0.266034, train/ssim=0.74711, validation/loss=0.287855, validation/num_examples=3554, validation/ssim=0.721337 +I0916 07:12:07.039201 140450009945856 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.17569, loss=0.349925 +I0916 07:12:07.042347 140525531075776 submission.py:307] 4500) loss = 0.350, grad_norm = 0.176 +I0916 07:12:31.201134 140450001553152 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.0986222, loss=0.286517 +I0916 07:12:31.204224 140525531075776 submission.py:307] 5000) loss = 0.287, grad_norm = 0.099 +I0916 07:12:55.367527 140450009945856 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.0287026, loss=0.235096 +I0916 07:12:55.370703 140525531075776 submission.py:307] 5500) loss = 0.235, grad_norm = 0.029 +I0916 07:13:19.579337 140450001553152 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.073794, loss=0.274073 +I0916 07:13:19.582439 140525531075776 submission.py:307] 6000) loss = 0.274, grad_norm = 0.074 +I0916 07:13:43.753641 140450009945856 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.187998, loss=0.210856 +I0916 07:13:43.756906 140525531075776 submission.py:307] 6500) loss = 0.211, grad_norm = 0.188 +I0916 07:13:47.844422 140525531075776 spec.py:333] Evaluating on the training split. +I0916 07:13:49.802844 140525531075776 spec.py:346] Evaluating on the validation split. +I0916 07:13:51.783440 140525531075776 spec.py:363] Evaluating on the test split. +I0916 07:13:53.765060 140525531075776 submission_runner.py:516] Time since start: 2746.43s, Step: 6574, {'train/ssim': 0.7507352828979492, 'train/loss': 0.2638390064239502, 'validation/ssim': 0.723855657797552, 'validation/loss': 0.2865405664996747, 'validation/num_examples': 3554, 'test/ssim': 0.740998703552604, 'test/loss': 0.28798414974431025, 'test/num_examples': 3581, 'score': 1830.8125085830688, 'total_duration': 2746.4348096847534, 'accumulated_submission_time': 1830.8125085830688, 'accumulated_eval_time': 896.7056200504303, 'accumulated_logging_time': 0.48372697830200195} +I0916 07:13:53.785876 140450001553152 logging_writer.py:48] [6574] accumulated_eval_time=896.706, accumulated_logging_time=0.483727, accumulated_submission_time=1830.81, global_step=6574, preemption_count=0, score=1830.81, test/loss=0.287984, test/num_examples=3581, test/ssim=0.740999, total_duration=2746.43, train/loss=0.263839, train/ssim=0.750735, validation/loss=0.286541, validation/num_examples=3554, validation/ssim=0.723856 +I0916 07:13:54.353507 140450009945856 logging_writer.py:48] [6574] global_step=6574, preemption_count=0, score=1830.81 +I0916 07:13:54.529684 140525531075776 submission_runner.py:857] Final fastmri score: 1830.8125085830688 diff --git a/logs/self_tuning/ademamix_golden/study_0/fastmri_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_0/fastmri_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..b9190214 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/fastmri_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,16 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/loss,test/num_examples,test/ssim,total_duration,train/loss,train/ssim,validation/loss,validation/num_examples,validation/ssim +789.4131972789764,0.0,290.30148816108704,1,0,290.30148816108704,1.076364542245532,3581,0.2514865239283719,1080.350521326065,1.0799249921526228,0.2339261770248413,1.0797452309457654,3554,0.2287525355178584 +797.2729263305664,0.0454843044281005,399.7377872467041,265,0,399.7377872467041,0.3197439748193591,3581,0.7065782842606814,1198.9923601150513,0.2963003771645682,0.7135448455810547,0.3176315556635129,3554,0.6889925276756823 +804.0663478374481,0.0765845775604248,511.3699195384979,328,0,511.3699195384979,0.3135263655512601,3581,0.7138226681400097,1318.5673644542694,0.2902473381587437,0.721343994140625,0.3113961120502427,3554,0.6961671295767093 +811.7989797592163,0.1067264080047607,621.8826193809509,395,0,621.8826193809509,0.3116902998791364,3581,0.7155941705136484,1437.9666678905487,0.2890457085200718,0.7220333644321987,0.3094093610412563,3554,0.6983517553944499 +819.5625236034393,0.1371378898620605,732.0183448791504,460,0,732.0183448791504,0.3090552037336812,3581,0.7179462653370218,1557.0150799751282,0.2857875483376639,0.7258890015738351,0.3066876808041995,3554,0.700805045811058 +827.3092002868652,0.1681602001190185,841.8979823589325,526,0,841.8979823589325,0.305638768958566,3581,0.7223764529810109,1675.794003725052,0.2826956680842808,0.7302299908229283,0.3035350793120427,3554,0.7050498228228756 +835.0769908428192,0.1985435485839843,952.068657398224,599,0,952.068657398224,0.3053292469151424,3581,0.7207503032497906,1794.8776230812073,0.2828923463821411,0.7273873601640973,0.3033684948957161,3554,0.7033286108082443 +843.3068838119507,0.2318663597106933,1062.5406551361084,667,0,1062.5406551361084,0.3032228266916538,3581,0.7256016181321558,1914.760534286499,0.2799873181751796,0.733936037336077,0.3010223680623944,3554,0.7085141604310284 +851.1057827472687,0.2652225494384765,1173.0753722190857,731,0,1173.0753722190857,0.3023471315624127,3581,0.7259842255567579,2034.258635044098,0.2794912202017648,0.7337468692234584,0.3003423945180606,3554,0.7086373298572032 +859.098197221756,0.296309232711792,1284.2708780765531,800,0,1284.2708780765531,0.3010443778252408,3581,0.7276828470792027,2154.6357412338257,0.2779566901070731,0.7359416144234794,0.2989743071187394,3554,0.7105753420441756 +866.6306080818176,0.3343243598937988,1394.8430495262146,864,0,1394.8430495262146,0.3002523013713872,3581,0.7278560839761938,2273.898585319519,0.2772273676736014,0.7358097348894391,0.2981984015589828,3554,0.7106857342738815 +875.186728477478,0.3900284767150879,1504.4105689525604,935,0,1504.4105689525604,0.3014664936252094,3581,0.7269426530647863,2393.2043352127075,0.2781445639474051,0.735457352229527,0.2993945120155107,3554,0.7099429395135762 +882.5525469779968,0.4228317737579345,1613.449716091156,2053,0,1613.449716091156,0.2930827413781067,3581,0.7350217238114354,2511.1351025104523,0.2696697541645595,0.7436562946864537,0.291437341178074,3554,0.7178362243686691 +890.7848875522614,0.4526228904724121,1722.1367993354795,4314,0,1722.1367993354795,0.2893549097013753,3581,0.7384776669619171,2629.9385311603546,0.2660341433116368,0.7471103668212891,0.2878547458079804,3554,0.7213369701172974 +896.7056200504303,0.48372697830200195,1830.8125085830688,6574,0,1830.8125085830688,0.28798414974431025,3581,0.740998703552604,2746.4348096847534,0.2638390064239502,0.7507352828979492,0.2865405664996747,3554,0.723855657797552 diff --git a/logs/self_tuning/ademamix_golden/study_0/fastmri_pytorch/trial_1/events.out.tfevents.1789540087.d4b46661c922.38.0.v2 b/logs/self_tuning/ademamix_golden/study_0/fastmri_pytorch/trial_1/events.out.tfevents.1789540087.d4b46661c922.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..e600e4623c9bab66d07c59019e112123accca5e9 GIT binary patch literal 39542 zcmbW=cOaGf9|v$EIfn`nGrkPTV^L`lhsUB)wZWU7HS1W^!2FZwZYz?} zNo*)?JjpvSz}Gt>AOshqWjzrWbLzA5Z3x=lUNA^oK-)sMVv-w}su3Hz$GB8y?{o9sx%>|7YSC_&r>M}DL>P%XiB+pt!3(} z`2Xqa8yQx;?Z0!(mFD+{9NPi^_Oz|Zc~k74Ay&?jFgcs-KBGajNO{lZqByUltaU`^>*hJh^GW7rdJ`m!RS^RT^5`!+C7oPbr*LNYf8oihD?|c5Ig}A|Cncr@crMcaeIB#BJ(D>#jAZb-!1W|aGH>gdl9*pO!F%^dr{wad<-bUkCF&sg$*haRQm>l;>#x^Mj_8tcrTla zijygeQ(!NSRg!6b1!pftDW~lRg|5i>_X30!v8XuxU1%Y*Myl)}g>vWL3;UMWM{BQc&z` zL8kc?oV_S+axN4UN_r$hSdoQ_17EdEKp|({hfy@(Du%RYDasx4)If1+AerV@a2^+< zX3uN}3f18xLRe9Nial~4v1uza>==b2SMhNO>v3^ozehAE9vvgoQYw^w__%?h%_R~c ztdQvg9i^|4_y+dk)#E{oLc~>!=*fCqyq%u24ir0Il4*Vg=W%f+RJjxsE5DEkVTB4R zR{L%}35p)}E{sCVRg^DaUG>f?JGBPIOJ_38ui!i`jy!E-1q$79Btlrx8Wm>!tZ#$D zx1KwrP~s{woLTn}J7aeK07aYDWSU>Wd0g0UIp7M4<%T3eSYd*Sf4p}M14TvJXhxyT zRg6xZj2+e4T+t2`na9X9zv3smX$ORqjseBmb0k7oVS|b@aa)&wqN$7zqiD!gWGc-- zMOE3I6`*L+m`w94ID4V;ZRspfEYctm!U`u;OvoOv1r*UM{TW3gt|E1vKPpU|Zs~x+ zIfYD1sW=pK6FW+IH;E8dj73HF;2$kOq0lgxQ8eZ%j9-VLB2({6BT!uLNT#J!ywE8* z3w!arJBbihgrcJD?BpMyu$mvvC{(x#$8BDykZ)DG925>oWSU>W=_rk+PuGCrNd}1! zR>Y#>=8bO3pqSY&no+276}ieTsEBZ}=?{uigUK|%g7fHpHNM9ZP;~Mp5yFZEsMxZm zNi-;2rcP%RYFvfB+i+Bzh>K1FMc6zt&9C4*x-VVagO7`k+ew75A|4e-cIKS}MPvIp zjG_rw;X82%Dq8B@*$s*WH!{ty;G9w8EAt=1UM%z`5yFbKs0bV&w;B`==PqOv>Rg4t zMR!!RTC4UJ6mwUSX(<(BCePCa#fyz3LRgW6isJ2eGeDslwv16UDU4G;{7W!Eu|tc zaq2Ws94;jh!io%3JUKoj9~6me6B&gDS20J{5*1FC&u4?eK7~y4E55VuK#bczjQ~aG z10+INaTFC+_s2a1g+bdSMxn`7yt>#46(>_a<13W2EXg##g7fGu-1fr@6sChngs>tD z75y5uKLUz&bG9*x<~)jl=BP;SDH{oU;jxiS^D8(_yT;1nCnzlTkO*N#4k~&+ZQK(S z_l$NjiWXc&@Q8M(uy`951PY@bWSU>W*^5Wzms)~il^ux?R$NC#c+VG>pji4kgHdR4 z74NGWqGIOdSbSXU_)4bv6`aS#x!`bou6L>azTXQFR@_6yAUWHAKvB=}FcYH9RkS&x zj0%+)|M##LD?G?FzoL@;=vJII<2&qymLG`_RurJ3rTcCxP{iKNWE3sAis9!Pqr&`x z-ZfA}7LjRw#W%LXL)m{iD0Iq5gs|cjDkjM!CxfD=?-@qXimRBGu8WEb8&^#SMP4|W z=2uj(6+ung&IX14G!h}K_=t)gpGyXULhVd8qtM|hzCQhl?*$dORLlXzgj-}<{Z+96*Tdrd3g)ewUCBNzO5fp(9@_$eBE6Ui4s{0%870v`L5+STmM8$^G zPNkp_pLoQqqa9cAW2GAI#l_P*@QgZlk4#IcxHI1TBh~{#;_?C#A*|3w#mn{Y3T5YDLi56LvY zg7dgo+pBaXDDp~3gs`GDDr7>MrGO&Wu!>O_a21(;Z*VX2R1;M}an_Vf^DEx5_hRms zC>v13+K~uhg)S;iy*eWU3dNRkdDZXQ8*&wKuV0}e@mLclP~7WCrlnLAs<&$b3TtZ; zA*|3xg`Y;R?Vxb!sK_Wfauq{t?xAA9wdH3)k!DJ!`4uJXy|_~8;Q|WBek4Lz(Fqk7 z)^yweilr|aGKx-IMZIS~P+_9YdjH~kDVgS1ykRTM;ysUp;%L1_zZW2^FhNC-eozP~ zY|7P`kj`91%iLS25HD_&4SOM1zsc`ue#L9HVpef4ZBU$RL?VP0rl{CxIN1;sYldku z>oDRfW_w&ig=NIGZ=iVXO{S$(M2+%D0!9Bo5+SVUfr`iHbSpsNcBdtyFy<;g7-gX% z;n=0)phzns(^4v~AGxXz3cL3tLReviiuiwIAA>^Ub6ZAX!d1*V#!^gbu5=0%##LmR zU%`2FPfnQV3<_EGcE1-Otgu1F!JH@fIS{YW`bu28AKw4751o5OQ~NDioSQK4OaxH%|Z29argMWZ#WCt6LC0`U`f)~iW`u)-Y`*_WDz zgQBJEOh#eORZM=7iVFFKg_WRqAtuxO3cp2%F=Aqyi8`t{gTi6p zN=DIxt61COI4|VZ~TfOe&Uf1jTvJBu3Gb ztC*aA0GFWNJoP>(o(7U>0mYUb7*X%xVSJXtcFP>h?K!6+=aiWW;x z;u1bT5C?)HI-X4PE6&UA#)y!U}&>xX(4gPqRFIdW=z6auvnHS+CRT z&6|O*4ce8EX@123-OZ@jV|%_nD2{(45yFZ~iTqP-HJ6)BFlWd)6hVkw=LJC{C;* z5yFaDsBpHacm|4`8!s3|U+!KUewu^_@}lw3EKm%8M5g%_ns*W~qSr>9y`YeLOCp37 zb5QXjPvI0OM!YIv6#cl0$-8#q5^@#)9S({irDU34k=AH7Ms)GorU?q)R&RbUKv*#k z6&B4C@fFUnbEQm3f36}&k9GU7^V;zuP}trk)BK7zZVON`Q^UL=DD0n*2w}wnR1BSX zaWL#f)uSp#VarwQ(Pcd}-*2w?35un!$uz%0*=If~yh8T{fx@JWL~EiHkMAWdI7+7Z6^r|fLWP_A$;Y5*b(Ta3D^{Q)*>Kcw*o&RlO&EnESCO1{ z0+*mLW}OV|MYlpS&9B&9FbpFMzcuUv3jenxLRhg972X>=;G;YEyal5e$W zpYvcVC=T2r)BK8@@m{F#84;HPiVuY(LRhgH6*)G8)L<`;#o027L0rY>oISXN5k8mk zUHi=SWSUj14~nvM5+STui;5<ZQ4HlOu77)mOK4iS=qv2Sy$xiV zUvYXBYcB%MY01D|?Al2pgcX}m(ZM$epCxSw4rCNgTt&Nqti4EGnOq+feW#LXenm;v zKX@QK(lu2<5wVIy2rH6M@sH8QR&Yk$O^;?2&RoSo`{lR<(YX?DQ0QlpX@12CrRf;4 zJpBPaW3{|MB7_yms7OIjNx-8qcHg{uf%!FrF?y>G=opg3+qruh{CAJ|RX@2(T< zMH?FuA*|Sn3UmElJ3z4{F@aGG<0`(cXT3X_YY-6!iouy=nqM*LsU_~kK~=*_Q1re) zB7_y&QPDliyFMt6Oy9*QT)B$yf41Qg`i^|o9~8|NkZFF!NySYV5mf&wek!=-3KAi# zNJGVj8)gSUaWLmNqZrOr$e(1rJo&IBI0_V3?vZJJ#U2gTebkM|kJrFn40=W)gca$i zC`dJE42otNml=f{SJC=%1|CTLt=I9hYTMhBX#vH`MHn&UmJ+@;s9;ATgcZ9`(by~_ z6cl!=ZZV1xTt&f=Gq{BO<_C|$UVPb1ruh{<<*b*q{EyyCgS}Y2mqZ9F_Mjrnps^w- zwwgU;6z*Jw+9KBLQA-}DTm;2WTQbeB=+}q!3TMT-%2-ge_8}3%iVRfv4)N~-ipti- zjAA5LVLJU3?!{Fdqwk>DZ%C&363lbr$*pG_DDn0xgu}{ZJ8HEQ| zp}vk)FuMBvlR@$PBAMn_7+hM12f{r1Ail5Xc9TR1D-NP!k@GDl*o&CwKN!U*u3|&p zAzXsi8rO%Q(E3cK`4y(ujWA-VoIwTb#hRZaLRfJa72BF`E(3*kkmy?VYapIn#m0;q zxP;SzM@>LsH-${|E5eM|V}$sm>>yAiEFuxYileCLpx!3}6f?f5Gm6n%g;B!-TtcJC zo#%ieTCVBuX?{iO7Z;3ZzV-4)P%Ks>5yFb&sJIf`4gW65%&G;mjxk(Cj6LfPWJvK_ z{MxcuM>5T?cxc6XqSgDhZzL!p#*qkN#R*hc1TJ0gf5xNjIJdPj6%)-jf=*r?6=eAKO85Af5E z-Di+#enpJVU{owrP_h8Uk0m5RSdoQ_z_qtZKoQ@?h*5ZR6_e(%UZ+*tn?ze7K5UuUPNGjI&wT9TW><$+Uo?5$m1e z!7_K6gW}{e5+SU}Mn(8NH~g$x?vehC!k4R0LK8X-k7OC^nd`eqDXlTUbx%3mOa3&G19>+v7`&Af&h#Z;0d<2kt$% z208YMvC8mWaocTaNNr!yc-`9sm(%4)@kdCyHApx~a!rXz$+#w`D91?nTD=%on)Fv+ z()ilBdo+BPrI{y9tloIew=_CEGew`CkKqo;i1p#ikd#e1#CkEa>Cbhot!lncFXu># zi)i0Ono-B^Y5cY3$z_w8tW{sGdx;+WOS8CN)EN5YB7RY0p9wS4;VU)oPp0#=d{g6@ zr>pQ=5GP*WrC?XHkg)E@wu#(k&>(5Qs*$t2Rd@QB+0>-!YkXcMYPNu;{`tEaCp!U{0zRjvtScVsoU2zmSxD}Yah0sV;U^`?`vOk1Zzbmy$7D>TNB2HL#=pM@rFpA z=!4g@K3HV_F8@4~flB=;WF@uLO1&OkaXG$I(v2WVTUj_sa!pPlth>v9x95aIt(R2mx zOH3qDOf8qTX7$j z^?$DnwNma$Q|eYLD&d7+pb;j{bJeS^l|^zwcPmEutXeS`|D+X!D8}=KNS^3Hr2gB>dY0YUQ7sG)UT7;Vf_ekVgtoD-I87 z>Yr<+ZJ8sk^3CkM51>{CJ)?Qj)k?wxtd;ES4(?DZol9t>V6DVnz*<>v*6<&wmF1sk zO5JK@UAuSqH+_a_8vUwkC0DRk9^dMTw+F#ySv#RtrtRYmkv!2)yR5(`9J9MQIzkzo zJ4!=ptCjKz-EcXfAByn4CoD@iNpejc6*l6UazanZK&?E#NR$4mRt`3PiFX=ZuF*vP zT5-L?s+APWn9fiuT6g&}BxO^rScSQz@ZnadmD!JJMjdPAwwC=Is1@ZGG>cm+on_7O zai;q*{{+;E(px%TORY>zI*UPZ`!93LDJR=XL*_>T7#fg+E&ukKi7&((I8x< zW{#*8)Jj)5P4bYFs#Xeul<_=WXL~;hYUPNSMhe!-|6k&Qr&4F^mRLcpD5%kty46bH zxh1FI_Y0EWJswtFE1!A3D81dUSoBmD^bCbF{A@WLuYQzpe!)W&!upKsq8D?mPEa=CGwKVCk`fH7DKJbx1mXYRVy=13>QNsbl0be{I#Mvn^h}! zS|5H789R*mG9+a!U0Jo_p!seK)QZTQX4J7(a;EiO0k!hJ7tP|2V$)h_;^2oTG92Pk%G07)f5+IXuQxDYGvX?no_q~`SNS;3}}Qy^XG4< zu9fxmh3-}?pR>L#H!I(^1VR$xc|#;m^cfz@@X+kf-WCdFpt+WY)K)9T-mI^K9(XP` z9FiJs7EY2}(|VmmTvPObv)7UV)>NjNmVOj zgIRw|B@!k6fLi%gOe2MBr9B?L@g|oiL#+&XPgCkvE6ED`@sC&zT2DA$T`La-Yvt}; zR;_$&Us(pVGR1*6MDj$BZqBL|$JqV4PzDuFG^Dm#=^MbRl{fF#zlWq*?!rluYl_Kc zeUNb2pzjT!R^-Ogq`#__tjV&;PzhEOXd-{Dw0OX(l|?$mQIIh*m@h+8W}U|>%sJEB zkAzxDiKH2Itd(vf4@`wxSu&Moacd`CW9udT9IEe&JJp2 z+-I8fSGCeY_4XO41p7*w$X_dg4y@;#{xa{6LPli0Sh8m{LsB-aEvr_1uGWu$TA3!I z8Fj3c-c^m#p;oF?Xco6tb}mT5)A@s<$Q^3shX$RmrB*tnE8s7-Y8S=2L9N8KqCwKu z3TJs&cLqm7t@!BC)IZmXL$(`oCn~>kgjx}Irg>7;%DqFZ53fCMvG6C<%4bs=DOf8q z_PDTZ$2M++S{YzPQ|eYLTh6){KqENVJC#@0N`YXlG#JCGmG^U>;qOD8bmk3_Jkd{o zT!OV?etTLal!5w48d6)W#C~AaO8nk`RUj#LtZ91x-_eT3I&WjxJ=RUggV>l7vC ze>JETlg2J&%l=#|o@wsLEi36g5NhRlGnyw=t;Ba?eOHTn{n>+{R{zvO(|39|ekM(t2L(jM$hFVGQPa_3uB`W|Imab?S1hw*I zAWf-Tt*mM`OdsAUcXFMsQ(Y^scz!xra~kW9DVpfzT!b@xNDLjXew1+DDc_d<89$U1 zop$##9FraMXolc1k%_@$k|R<(3Xu6O75deXM?AlEd$?*Y-Ycz>?}qQLq>Q5D{1EpO zA(&CrK_M0(I=(bR5E7e-kfol-J^@6pUOnS#$aS6{C$$JXoes}<~u?bJ*!4*pg89aRQT{Dc~igj=oO{wD*OC|3<^y7v_oz1IjgV`9TPrCi^fxCM}T!!3JNLyq$Ns;Fn&LVTDV{da)| zKtjjRaem0N+ZLGdE+l;h6#cF7G(!*)Yl)kiu4dp0kaQ)7e$|j&JU;_D^abnrMDYG1 z+>s)6I?fLE{DP_x4DKYDf~#@Ag$}W?g0us+d>6 zjs!oZ()Su~h~&j!OLx}CAUB$Cwt@@hy1O)_wo7l^4%X*i44Sl#horNQg_9)LwBMWc z7bLA7znu=3-li{T(qCPAul=&gfSs3nOB1Vip7Xb*PnCP&;l46hSr#(%%lI-RWv=a6 ze{%P-(c=?v>FxHNX4LW08?<-o8o2a2$qpmu5cksS-9I0%MeADaT?&`py^3_cmP@a^ zmoA>ncjkUN1()7ljcJgymtM~D-WxBt1DD=G8Z`CKmtL#=W03nix#J_a^fqrr^Q5}; znw)xx?cD9~oJG(DC3-YcunVrw!^5XlJPO~4|JjMA)a}x{Qh&`P=*Ru*&pKDv%1WN! zZ}c``eYa>k=fqSv!#i%HD59vqBnHQz6lT|G0hN!*la+^p}hP902!W? zKe`(7gXeeN6i$ZX(X>%9$EOTwJLot+B)82QEBM&{_4?;^0dim;%@BkX>_o_wugf<9 z#G~^IpK8cQo*(AwJ!~;Tj251m0XvdlM#uRfu5H%fjy!(#wmm?eSknwaNNgrT?hk+R z4In4oJ_b}niggeb;c+y)T4v2>gtGH;d*X5{Tl(+0>Cf0`i(X?+DDpJnCn z%ZOL`h6Pqb_H+}vhcGd*!Hb%PVVWP@Lre&1 | tee -a /logs/finewebedu_lm_pytorch_09-11-2026-18-40-54.log +W0911 18:40:56.999000 9 site-packages/torch/distributed/run.py:803] +W0911 18:40:56.999000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0911 18:40:56.999000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0911 18:40:56.999000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-11 18:40:58.602540: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-11 18:40:58.602535: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-11 18:40:58.602537: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-11 18:40:58.602538: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789152058.623338 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789152058.623339 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789152058.623342 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789152058.623521 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789152058.630272 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789152058.630270 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789152058.630272 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789152058.630518 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789152058.647842 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789152058.647842 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789152058.647843 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789152058.647843 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789152058.647864 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789152058.647864 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789152058.647867 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789152058.647866 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789152058.647867 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789152058.647869 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789152058.647869 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789152058.647869 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789152058.647869 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789152058.647872 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789152058.647872 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789152058.647874 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789152064.831392 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789152065.013069 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789152065.039855 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789152065.131356 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W911 18:41:07.856316316 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0911 18:41:08.281187 139763972338880 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/finewebedu_lm_pytorch. +I0911 18:41:08.281187 140504811476160 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/finewebedu_lm_pytorch. +I0911 18:41:08.281187 139767878051008 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/finewebedu_lm_pytorch. +I0911 18:41:08.281210 140410517628096 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/finewebedu_lm_pytorch. +I0911 18:41:08.304763 139763972338880 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/finewebedu_lm_pytorch/trial_1. +I0911 18:41:08.562975 139763972338880 submission_runner.py:242] Initializing dataset. +I0911 18:41:08.563141 139763972338880 submission_runner.py:251] Initializing model. +I0911 18:41:15.309361 139763972338880 submission_runner.py:290] Performing `torch.compile`. +I0911 18:41:16.813744 140410517628096 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0911 18:41:16.813756 140504811476160 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0911 18:41:16.813900 140504811476160 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0911 18:41:16.813902 140410517628096 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0911 18:41:16.814075 139763972338880 submission_runner.py:294] Initializing optimizer. +I0911 18:41:16.814785 139763972338880 submission_runner.py:299] Initializing metrics bundle. +I0911 18:41:16.814934 139763972338880 submission_runner.py:321] Initializing checkpoint and logger. +I0911 18:41:16.815514 139763972338880 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_0/finewebedu_lm_pytorch/trial_1/meta_data_0.json. +I0911 18:41:16.815703 139763972338880 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0911 18:41:16.815760 139763972338880 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0911 18:41:16.820118 139767878051008 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0911 18:41:16.820275 139767878051008 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0911 18:41:17.061790 139763972338880 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_0/finewebedu_lm_pytorch/trial_1/flags_0.json. +I0911 18:41:17.076434 139763972338880 submission_runner.py:359] Starting training loop. +[rank0]:W0911 18:41:18.985000 38 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank1]:W0911 18:41:18.998000 39 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank2]:W0911 18:41:19.022000 40 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank3]:W0911 18:41:19.023000 41 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +I0911 18:42:02.224758 139737750558464 logging_writer.py:48] [0] global_step=0, grad_norm=0.5, loss=10.9965 +I0911 18:42:02.301408 139763972338880 submission.py:307] 0) loss = 10.996, grad_norm = 0.500 +I0911 18:42:02.914943 139763972338880 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +I0911 18:42:57.093676 139763972338880 spec.py:346] Evaluating on the validation split. +I0911 18:45:47.797976 139763972338880 spec.py:363] Evaluating on the test split. +I0911 18:45:47.798639 139763972338880 submission_runner.py:516] Time since start: 270.72s, Step: 1, {'train/loss': 11.000336456298829, 'train/ppl': 59894.29013665645, 'validation/loss': 10.999931315936701, 'validation/ppl': 59870.02945708319, 'validation/num_examples': 100000, 'test/loss': 0.0, 'test/ppl': 1.0, 'test/num_examples': 0, 'score': 45.22602105140686, 'total_duration': 270.7221882343292, 'accumulated_submission_time': 45.22602105140686, 'accumulated_eval_time': 224.88363099098206, 'accumulated_logging_time': 0} +I0911 18:45:47.831084 139735811856128 logging_writer.py:48] [1] accumulated_eval_time=224.884, accumulated_logging_time=0, accumulated_submission_time=45.226, global_step=1, preemption_count=0, score=45.226, test/loss=0, test/num_examples=0, test/ppl=1, total_duration=270.722, train/loss=11.0003, train/ppl=59894.3, validation/loss=10.9999, validation/num_examples=100000, validation/ppl=59870 +I0911 18:45:49.548801 139735803463424 logging_writer.py:48] [1] global_step=1, grad_norm=0.5, loss=11.0073 +I0911 18:45:49.551675 139763972338880 submission.py:307] 1) loss = 11.007, grad_norm = 0.500 +I0911 18:45:49.989794 139735811856128 logging_writer.py:48] [2] global_step=2, grad_norm=0.5, loss=10.9815 +I0911 18:45:49.992761 139763972338880 submission.py:307] 2) loss = 10.981, grad_norm = 0.500 +I0911 18:45:50.423228 139735803463424 logging_writer.py:48] [3] global_step=3, grad_norm=0.5, loss=10.9541 +I0911 18:45:50.426172 139763972338880 submission.py:307] 3) loss = 10.954, grad_norm = 0.500 +I0911 18:45:50.852049 139735811856128 logging_writer.py:48] [4] global_step=4, grad_norm=0.5, loss=10.906 +I0911 18:45:50.854909 139763972338880 submission.py:307] 4) loss = 10.906, grad_norm = 0.500 +I0911 18:45:51.283553 139735803463424 logging_writer.py:48] [5] global_step=5, grad_norm=0.5, loss=10.833 +I0911 18:45:51.286563 139763972338880 submission.py:307] 5) loss = 10.833, grad_norm = 0.500 +I0911 18:45:51.714295 139735811856128 logging_writer.py:48] [6] global_step=6, grad_norm=0.5, loss=10.7457 +I0911 18:45:51.717133 139763972338880 submission.py:307] 6) loss = 10.746, grad_norm = 0.500 +I0911 18:45:52.142734 139735803463424 logging_writer.py:48] [7] global_step=7, grad_norm=0.5, loss=10.6298 +I0911 18:45:52.145547 139763972338880 submission.py:307] 7) loss = 10.630, grad_norm = 0.500 +I0911 18:45:52.573957 139735811856128 logging_writer.py:48] [8] global_step=8, grad_norm=0.5, loss=10.5443 +I0911 18:45:52.576819 139763972338880 submission.py:307] 8) loss = 10.544, grad_norm = 0.500 +I0911 18:45:53.009006 139735803463424 logging_writer.py:48] [9] global_step=9, grad_norm=0.5, loss=10.4063 +I0911 18:45:53.011955 139763972338880 submission.py:307] 9) loss = 10.406, grad_norm = 0.500 +I0911 18:45:53.438341 139735811856128 logging_writer.py:48] [10] global_step=10, grad_norm=0.5, loss=10.3104 +I0911 18:45:53.441161 139763972338880 submission.py:307] 10) loss = 10.310, grad_norm = 0.500 +I0911 18:45:53.869719 139735803463424 logging_writer.py:48] [11] global_step=11, grad_norm=0.5, loss=10.1627 +I0911 18:45:53.872570 139763972338880 submission.py:307] 11) loss = 10.163, grad_norm = 0.500 +I0911 18:45:54.301736 139735811856128 logging_writer.py:48] [12] global_step=12, grad_norm=0.5, loss=10.0831 +I0911 18:45:54.304547 139763972338880 submission.py:307] 12) loss = 10.083, grad_norm = 0.500 +I0911 18:45:54.731983 139735803463424 logging_writer.py:48] [13] global_step=13, grad_norm=0.5, loss=9.92597 +I0911 18:45:54.734800 139763972338880 submission.py:307] 13) loss = 9.926, grad_norm = 0.500 +I0911 18:45:55.161995 139735811856128 logging_writer.py:48] [14] global_step=14, grad_norm=0.5, loss=9.81596 +I0911 18:45:55.164871 139763972338880 submission.py:307] 14) loss = 9.816, grad_norm = 0.500 +I0911 18:45:55.594337 139735803463424 logging_writer.py:48] [15] global_step=15, grad_norm=0.5, loss=9.79638 +I0911 18:45:55.597197 139763972338880 submission.py:307] 15) loss = 9.796, grad_norm = 0.500 +I0911 18:45:56.023986 139735811856128 logging_writer.py:48] [16] global_step=16, grad_norm=0.5, loss=9.67793 +I0911 18:45:56.026865 139763972338880 submission.py:307] 16) loss = 9.678, grad_norm = 0.500 +I0911 18:45:56.451277 139735803463424 logging_writer.py:48] [17] global_step=17, grad_norm=0.5, loss=9.56807 +I0911 18:45:56.454164 139763972338880 submission.py:307] 17) loss = 9.568, grad_norm = 0.500 +I0911 18:45:56.881242 139735811856128 logging_writer.py:48] [18] global_step=18, grad_norm=0.5, loss=9.51242 +I0911 18:45:56.884105 139763972338880 submission.py:307] 18) loss = 9.512, grad_norm = 0.500 +I0911 18:45:57.313086 139735803463424 logging_writer.py:48] [19] global_step=19, grad_norm=0.5, loss=9.46872 +I0911 18:45:57.315822 139763972338880 submission.py:307] 19) loss = 9.469, grad_norm = 0.500 +I0911 18:45:57.743370 139735811856128 logging_writer.py:48] [20] global_step=20, grad_norm=0.5, loss=9.3479 +I0911 18:45:57.746214 139763972338880 submission.py:307] 20) loss = 9.348, grad_norm = 0.500 +I0911 18:45:58.173027 139735803463424 logging_writer.py:48] [21] global_step=21, grad_norm=0.5, loss=9.35899 +I0911 18:45:58.176019 139763972338880 submission.py:307] 21) loss = 9.359, grad_norm = 0.500 +I0911 18:45:58.602866 139735811856128 logging_writer.py:48] [22] global_step=22, grad_norm=0.5, loss=9.29358 +I0911 18:45:58.605807 139763972338880 submission.py:307] 22) loss = 9.294, grad_norm = 0.500 +I0911 18:45:59.035263 139735803463424 logging_writer.py:48] [23] global_step=23, grad_norm=0.5, loss=9.24652 +I0911 18:45:59.038172 139763972338880 submission.py:307] 23) loss = 9.247, grad_norm = 0.500 +I0911 18:45:59.466051 139735811856128 logging_writer.py:48] [24] global_step=24, grad_norm=0.5, loss=9.28879 +I0911 18:45:59.468868 139763972338880 submission.py:307] 24) loss = 9.289, grad_norm = 0.500 +I0911 18:45:59.894564 139735803463424 logging_writer.py:48] [25] global_step=25, grad_norm=0.5, loss=9.2383 +I0911 18:45:59.897475 139763972338880 submission.py:307] 25) loss = 9.238, grad_norm = 0.500 +I0911 18:46:00.323560 139735811856128 logging_writer.py:48] [26] global_step=26, grad_norm=0.5, loss=9.21029 +I0911 18:46:00.326418 139763972338880 submission.py:307] 26) loss = 9.210, grad_norm = 0.500 +I0911 18:46:00.755624 139735803463424 logging_writer.py:48] [27] global_step=27, grad_norm=0.5, loss=9.21307 +I0911 18:46:00.758482 139763972338880 submission.py:307] 27) loss = 9.213, grad_norm = 0.500 +I0911 18:46:01.186837 139735811856128 logging_writer.py:48] [28] global_step=28, grad_norm=0.5, loss=9.16776 +I0911 18:46:01.189640 139763972338880 submission.py:307] 28) loss = 9.168, grad_norm = 0.500 +I0911 18:46:01.616074 139735803463424 logging_writer.py:48] [29] global_step=29, grad_norm=0.5, loss=9.16667 +I0911 18:46:01.618903 139763972338880 submission.py:307] 29) loss = 9.167, grad_norm = 0.500 +I0911 18:46:02.045222 139735811856128 logging_writer.py:48] [30] global_step=30, grad_norm=0.5, loss=9.10725 +I0911 18:46:02.048123 139763972338880 submission.py:307] 30) loss = 9.107, grad_norm = 0.500 +I0911 18:46:02.477123 139735803463424 logging_writer.py:48] [31] global_step=31, grad_norm=0.5, loss=9.14014 +I0911 18:46:02.480018 139763972338880 submission.py:307] 31) loss = 9.140, grad_norm = 0.500 +I0911 18:46:02.907379 139735811856128 logging_writer.py:48] [32] global_step=32, grad_norm=0.5, loss=9.11178 +I0911 18:46:02.910269 139763972338880 submission.py:307] 32) loss = 9.112, grad_norm = 0.500 +I0911 18:46:03.337271 139735803463424 logging_writer.py:48] [33] global_step=33, grad_norm=0.5, loss=9.04429 +I0911 18:46:03.340099 139763972338880 submission.py:307] 33) loss = 9.044, grad_norm = 0.500 +I0911 18:46:03.767253 139735811856128 logging_writer.py:48] [34] global_step=34, grad_norm=0.5, loss=9.03998 +I0911 18:46:03.770131 139763972338880 submission.py:307] 34) loss = 9.040, grad_norm = 0.500 +I0911 18:46:04.197762 139735803463424 logging_writer.py:48] [35] global_step=35, grad_norm=0.5, loss=8.98115 +I0911 18:46:04.200580 139763972338880 submission.py:307] 35) loss = 8.981, grad_norm = 0.500 +I0911 18:46:04.629127 139735811856128 logging_writer.py:48] [36] global_step=36, grad_norm=0.5, loss=8.9003 +I0911 18:46:04.631973 139763972338880 submission.py:307] 36) loss = 8.900, grad_norm = 0.500 +I0911 18:46:05.059368 139735803463424 logging_writer.py:48] [37] global_step=37, grad_norm=0.5, loss=8.89531 +I0911 18:46:05.062202 139763972338880 submission.py:307] 37) loss = 8.895, grad_norm = 0.500 +I0911 18:46:05.488600 139735811856128 logging_writer.py:48] [38] global_step=38, grad_norm=0.5, loss=8.85006 +I0911 18:46:05.491429 139763972338880 submission.py:307] 38) loss = 8.850, grad_norm = 0.500 +I0911 18:46:05.919637 139735803463424 logging_writer.py:48] [39] global_step=39, grad_norm=0.5, loss=8.83963 +I0911 18:46:05.922558 139763972338880 submission.py:307] 39) loss = 8.840, grad_norm = 0.500 +I0911 18:46:06.349917 139735811856128 logging_writer.py:48] [40] global_step=40, grad_norm=0.5, loss=8.81861 +I0911 18:46:06.352711 139763972338880 submission.py:307] 40) loss = 8.819, grad_norm = 0.500 +I0911 18:46:06.778993 139735803463424 logging_writer.py:48] [41] global_step=41, grad_norm=0.5, loss=8.84278 +I0911 18:46:06.781920 139763972338880 submission.py:307] 41) loss = 8.843, grad_norm = 0.500 +I0911 18:46:07.208624 139735811856128 logging_writer.py:48] [42] global_step=42, grad_norm=0.5, loss=8.85248 +I0911 18:46:07.211542 139763972338880 submission.py:307] 42) loss = 8.852, grad_norm = 0.500 +I0911 18:46:07.639378 139735803463424 logging_writer.py:48] [43] global_step=43, grad_norm=0.5, loss=8.79128 +I0911 18:46:07.642249 139763972338880 submission.py:307] 43) loss = 8.791, grad_norm = 0.500 +I0911 18:46:08.070110 139735811856128 logging_writer.py:48] [44] global_step=44, grad_norm=0.5, loss=8.81408 +I0911 18:46:08.072968 139763972338880 submission.py:307] 44) loss = 8.814, grad_norm = 0.500 +I0911 18:46:08.500030 139735803463424 logging_writer.py:48] [45] global_step=45, grad_norm=0.5, loss=8.7221 +I0911 18:46:08.502940 139763972338880 submission.py:307] 45) loss = 8.722, grad_norm = 0.500 +I0911 18:46:08.930345 139735811856128 logging_writer.py:48] [46] global_step=46, grad_norm=0.5, loss=8.73811 +I0911 18:46:08.933158 139763972338880 submission.py:307] 46) loss = 8.738, grad_norm = 0.500 +I0911 18:46:09.361110 139735803463424 logging_writer.py:48] [47] global_step=47, grad_norm=0.5, loss=8.73262 +I0911 18:46:09.363938 139763972338880 submission.py:307] 47) loss = 8.733, grad_norm = 0.500 +I0911 18:46:09.790918 139735811856128 logging_writer.py:48] [48] global_step=48, grad_norm=0.5, loss=8.70247 +I0911 18:46:09.793769 139763972338880 submission.py:307] 48) loss = 8.702, grad_norm = 0.500 +I0911 18:46:10.220173 139735803463424 logging_writer.py:48] [49] global_step=49, grad_norm=0.5, loss=8.67884 +I0911 18:46:10.223124 139763972338880 submission.py:307] 49) loss = 8.679, grad_norm = 0.500 +I0911 18:46:10.650801 139735811856128 logging_writer.py:48] [50] global_step=50, grad_norm=0.5, loss=8.63155 +I0911 18:46:10.653668 139763972338880 submission.py:307] 50) loss = 8.632, grad_norm = 0.500 +I0911 18:46:11.079909 139735803463424 logging_writer.py:48] [51] global_step=51, grad_norm=0.5, loss=8.68553 +I0911 18:46:11.082773 139763972338880 submission.py:307] 51) loss = 8.686, grad_norm = 0.500 +I0911 18:46:11.508217 139735811856128 logging_writer.py:48] [52] global_step=52, grad_norm=0.5, loss=8.60084 +I0911 18:46:11.511097 139763972338880 submission.py:307] 52) loss = 8.601, grad_norm = 0.500 +I0911 18:46:11.938268 139735803463424 logging_writer.py:48] [53] global_step=53, grad_norm=0.5, loss=8.6264 +I0911 18:46:11.941141 139763972338880 submission.py:307] 53) loss = 8.626, grad_norm = 0.500 +I0911 18:46:12.366814 139735811856128 logging_writer.py:48] [54] global_step=54, grad_norm=0.5, loss=8.63744 +I0911 18:46:12.369752 139763972338880 submission.py:307] 54) loss = 8.637, grad_norm = 0.500 +I0911 18:46:12.797422 139735803463424 logging_writer.py:48] [55] global_step=55, grad_norm=0.5, loss=8.46224 +I0911 18:46:12.800253 139763972338880 submission.py:307] 55) loss = 8.462, grad_norm = 0.500 +I0911 18:46:13.226403 139735811856128 logging_writer.py:48] [56] global_step=56, grad_norm=0.5, loss=8.51831 +I0911 18:46:13.229241 139763972338880 submission.py:307] 56) loss = 8.518, grad_norm = 0.500 +I0911 18:46:13.653945 139735803463424 logging_writer.py:48] [57] global_step=57, grad_norm=0.5, loss=8.57827 +I0911 18:46:13.656745 139763972338880 submission.py:307] 57) loss = 8.578, grad_norm = 0.500 +I0911 18:46:14.082845 139735811856128 logging_writer.py:48] [58] global_step=58, grad_norm=0.5, loss=8.51476 +I0911 18:46:14.085729 139763972338880 submission.py:307] 58) loss = 8.515, grad_norm = 0.500 +I0911 18:46:14.511308 139735803463424 logging_writer.py:48] [59] global_step=59, grad_norm=0.5, loss=8.45858 +I0911 18:46:14.514149 139763972338880 submission.py:307] 59) loss = 8.459, grad_norm = 0.500 +I0911 18:46:14.940954 139735811856128 logging_writer.py:48] [60] global_step=60, grad_norm=0.5, loss=8.40233 +I0911 18:46:14.943780 139763972338880 submission.py:307] 60) loss = 8.402, grad_norm = 0.500 +I0911 18:46:15.371856 139735803463424 logging_writer.py:48] [61] global_step=61, grad_norm=0.5, loss=8.45324 +I0911 18:46:15.374675 139763972338880 submission.py:307] 61) loss = 8.453, grad_norm = 0.500 +I0911 18:46:15.801887 139735811856128 logging_writer.py:48] [62] global_step=62, grad_norm=0.5, loss=8.43485 +I0911 18:46:15.804831 139763972338880 submission.py:307] 62) loss = 8.435, grad_norm = 0.500 +I0911 18:46:16.230703 139735803463424 logging_writer.py:48] [63] global_step=63, grad_norm=0.5, loss=8.36662 +I0911 18:46:16.233562 139763972338880 submission.py:307] 63) loss = 8.367, grad_norm = 0.500 +I0911 18:46:16.660272 139735811856128 logging_writer.py:48] [64] global_step=64, grad_norm=0.5, loss=8.35069 +I0911 18:46:16.663102 139763972338880 submission.py:307] 64) loss = 8.351, grad_norm = 0.500 +I0911 18:46:17.091844 139735803463424 logging_writer.py:48] [65] global_step=65, grad_norm=0.5, loss=8.35641 +I0911 18:46:17.094705 139763972338880 submission.py:307] 65) loss = 8.356, grad_norm = 0.500 +I0911 18:46:17.522319 139735811856128 logging_writer.py:48] [66] global_step=66, grad_norm=0.5, loss=8.33245 +I0911 18:46:17.525141 139763972338880 submission.py:307] 66) loss = 8.332, grad_norm = 0.500 +I0911 18:46:17.952859 139735803463424 logging_writer.py:48] [67] global_step=67, grad_norm=0.5, loss=8.39039 +I0911 18:46:17.955671 139763972338880 submission.py:307] 67) loss = 8.390, grad_norm = 0.500 +I0911 18:46:18.382304 139735811856128 logging_writer.py:48] [68] global_step=68, grad_norm=0.5, loss=8.30402 +I0911 18:46:18.385169 139763972338880 submission.py:307] 68) loss = 8.304, grad_norm = 0.500 +I0911 18:46:18.813443 139735803463424 logging_writer.py:48] [69] global_step=69, grad_norm=0.5, loss=8.27834 +I0911 18:46:18.816382 139763972338880 submission.py:307] 69) loss = 8.278, grad_norm = 0.500 +I0911 18:46:19.244419 139735811856128 logging_writer.py:48] [70] global_step=70, grad_norm=0.5, loss=8.28062 +I0911 18:46:19.247375 139763972338880 submission.py:307] 70) loss = 8.281, grad_norm = 0.500 +I0911 18:46:19.674335 139735803463424 logging_writer.py:48] [71] global_step=71, grad_norm=0.5, loss=8.22634 +I0911 18:46:19.677233 139763972338880 submission.py:307] 71) loss = 8.226, grad_norm = 0.500 +I0911 18:46:20.104752 139735811856128 logging_writer.py:48] [72] global_step=72, grad_norm=0.5, loss=8.14752 +I0911 18:46:20.107662 139763972338880 submission.py:307] 72) loss = 8.148, grad_norm = 0.500 +I0911 18:46:20.536331 139735803463424 logging_writer.py:48] [73] global_step=73, grad_norm=0.5, loss=8.21035 +I0911 18:46:20.539249 139763972338880 submission.py:307] 73) loss = 8.210, grad_norm = 0.500 +I0911 18:46:20.965736 139735811856128 logging_writer.py:48] [74] global_step=74, grad_norm=0.5, loss=8.16009 +I0911 18:46:20.968638 139763972338880 submission.py:307] 74) loss = 8.160, grad_norm = 0.500 +I0911 18:46:21.395159 139735803463424 logging_writer.py:48] [75] global_step=75, grad_norm=0.5, loss=8.14442 +I0911 18:46:21.398075 139763972338880 submission.py:307] 75) loss = 8.144, grad_norm = 0.500 +I0911 18:46:21.826696 139735811856128 logging_writer.py:48] [76] global_step=76, grad_norm=0.5, loss=8.15403 +I0911 18:46:21.829882 139763972338880 submission.py:307] 76) loss = 8.154, grad_norm = 0.500 +I0911 18:46:22.257148 139735803463424 logging_writer.py:48] [77] global_step=77, grad_norm=0.5, loss=8.08907 +I0911 18:46:22.259984 139763972338880 submission.py:307] 77) loss = 8.089, grad_norm = 0.500 +I0911 18:46:22.686375 139735811856128 logging_writer.py:48] [78] global_step=78, grad_norm=0.5, loss=8.04899 +I0911 18:46:22.689214 139763972338880 submission.py:307] 78) loss = 8.049, grad_norm = 0.500 +I0911 18:46:23.115890 139735803463424 logging_writer.py:48] [79] global_step=79, grad_norm=0.5, loss=8.05287 +I0911 18:46:23.118823 139763972338880 submission.py:307] 79) loss = 8.053, grad_norm = 0.500 +I0911 18:46:23.547844 139735811856128 logging_writer.py:48] [80] global_step=80, grad_norm=0.5, loss=8.03002 +I0911 18:46:23.550737 139763972338880 submission.py:307] 80) loss = 8.030, grad_norm = 0.500 +I0911 18:46:23.977715 139735803463424 logging_writer.py:48] [81] global_step=81, grad_norm=0.5, loss=8.0331 +I0911 18:46:23.980611 139763972338880 submission.py:307] 81) loss = 8.033, grad_norm = 0.500 +I0911 18:46:24.407056 139735811856128 logging_writer.py:48] [82] global_step=82, grad_norm=0.5, loss=8.03717 +I0911 18:46:24.410187 139763972338880 submission.py:307] 82) loss = 8.037, grad_norm = 0.500 +I0911 18:46:24.836998 139735803463424 logging_writer.py:48] [83] global_step=83, grad_norm=0.5, loss=7.93711 +I0911 18:46:24.839935 139763972338880 submission.py:307] 83) loss = 7.937, grad_norm = 0.500 +I0911 18:46:25.268724 139735811856128 logging_writer.py:48] [84] global_step=84, grad_norm=0.5, loss=7.88457 +I0911 18:46:25.271585 139763972338880 submission.py:307] 84) loss = 7.885, grad_norm = 0.500 +I0911 18:46:25.700227 139735803463424 logging_writer.py:48] [85] global_step=85, grad_norm=0.5, loss=7.95179 +I0911 18:46:25.703006 139763972338880 submission.py:307] 85) loss = 7.952, grad_norm = 0.500 +I0911 18:46:26.130257 139735811856128 logging_writer.py:48] [86] global_step=86, grad_norm=0.5, loss=7.88491 +I0911 18:46:26.133163 139763972338880 submission.py:307] 86) loss = 7.885, grad_norm = 0.500 +I0911 18:46:26.561942 139735803463424 logging_writer.py:48] [87] global_step=87, grad_norm=0.5, loss=7.98065 +I0911 18:46:26.564891 139763972338880 submission.py:307] 87) loss = 7.981, grad_norm = 0.500 +I0911 18:46:26.991867 139735811856128 logging_writer.py:48] [88] global_step=88, grad_norm=0.5, loss=7.91532 +I0911 18:46:26.994719 139763972338880 submission.py:307] 88) loss = 7.915, grad_norm = 0.500 +I0911 18:46:27.420949 139735803463424 logging_writer.py:48] [89] global_step=89, grad_norm=0.5, loss=7.86703 +I0911 18:46:27.423874 139763972338880 submission.py:307] 89) loss = 7.867, grad_norm = 0.500 +I0911 18:46:27.851954 139735811856128 logging_writer.py:48] [90] global_step=90, grad_norm=0.5, loss=7.87383 +I0911 18:46:27.854859 139763972338880 submission.py:307] 90) loss = 7.874, grad_norm = 0.500 +I0911 18:46:28.284431 139735803463424 logging_writer.py:48] [91] global_step=91, grad_norm=0.5, loss=7.83459 +I0911 18:46:28.287254 139763972338880 submission.py:307] 91) loss = 7.835, grad_norm = 0.500 +I0911 18:46:28.714359 139735811856128 logging_writer.py:48] [92] global_step=92, grad_norm=0.5, loss=7.81115 +I0911 18:46:28.717122 139763972338880 submission.py:307] 92) loss = 7.811, grad_norm = 0.500 +I0911 18:46:29.144162 139735803463424 logging_writer.py:48] [93] global_step=93, grad_norm=0.5, loss=7.74458 +I0911 18:46:29.147131 139763972338880 submission.py:307] 93) loss = 7.745, grad_norm = 0.500 +I0911 18:46:29.576316 139735811856128 logging_writer.py:48] [94] global_step=94, grad_norm=0.5, loss=7.77569 +I0911 18:46:29.579218 139763972338880 submission.py:307] 94) loss = 7.776, grad_norm = 0.500 +I0911 18:46:30.007078 139735803463424 logging_writer.py:48] [95] global_step=95, grad_norm=0.5, loss=7.7814 +I0911 18:46:30.010081 139763972338880 submission.py:307] 95) loss = 7.781, grad_norm = 0.500 +I0911 18:46:30.436945 139735811856128 logging_writer.py:48] [96] global_step=96, grad_norm=0.5, loss=7.77128 +I0911 18:46:30.439833 139763972338880 submission.py:307] 96) loss = 7.771, grad_norm = 0.500 +I0911 18:46:30.868331 139735803463424 logging_writer.py:48] [97] global_step=97, grad_norm=0.5, loss=7.69231 +I0911 18:46:30.871290 139763972338880 submission.py:307] 97) loss = 7.692, grad_norm = 0.500 +I0911 18:46:31.300275 139735811856128 logging_writer.py:48] [98] global_step=98, grad_norm=0.5, loss=7.79662 +I0911 18:46:31.303206 139763972338880 submission.py:307] 98) loss = 7.797, grad_norm = 0.500 +I0911 18:46:31.729624 139735803463424 logging_writer.py:48] [99] global_step=99, grad_norm=0.5, loss=7.73477 +I0911 18:46:31.732521 139763972338880 submission.py:307] 99) loss = 7.735, grad_norm = 0.500 +I0911 18:46:32.161160 139735811856128 logging_writer.py:48] [100] global_step=100, grad_norm=0.5, loss=7.69495 +I0911 18:46:32.164736 139763972338880 submission.py:307] 100) loss = 7.695, grad_norm = 0.500 +I0911 18:49:19.000717 139735803463424 logging_writer.py:48] [500] global_step=500, grad_norm=0.5, loss=5.60618 +I0911 18:49:19.004961 139763972338880 submission.py:307] 500) loss = 5.606, grad_norm = 0.500 +I0911 18:52:48.075905 139735811856128 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.499999, loss=4.56653 +I0911 18:52:48.079109 139763972338880 submission.py:307] 1000) loss = 4.567, grad_norm = 0.500 +I0911 18:56:17.163715 139735803463424 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.468964, loss=4.18384 +I0911 18:56:17.166995 139763972338880 submission.py:307] 1500) loss = 4.184, grad_norm = 0.469 +I0911 18:59:46.253277 139735811856128 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.333149, loss=3.88318 +I0911 18:59:46.256353 139763972338880 submission.py:307] 2000) loss = 3.883, grad_norm = 0.333 +I0911 19:03:15.304781 139735803463424 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.322272, loss=3.8495 +I0911 19:03:15.308016 139763972338880 submission.py:307] 2500) loss = 3.850, grad_norm = 0.322 +I0911 19:06:44.326397 139735811856128 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.27576, loss=3.64443 +I0911 19:06:44.329654 139763972338880 submission.py:307] 3000) loss = 3.644, grad_norm = 0.276 +I0911 19:10:13.307483 139735803463424 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.240318, loss=3.50689 +I0911 19:10:13.310672 139763972338880 submission.py:307] 3500) loss = 3.507, grad_norm = 0.240 +I0911 19:13:42.322683 139735811856128 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.247899, loss=3.56549 +I0911 19:13:42.325902 139763972338880 submission.py:307] 4000) loss = 3.565, grad_norm = 0.248 +I0911 19:17:11.320040 139735803463424 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.218377, loss=3.5659 +I0911 19:17:11.323283 139763972338880 submission.py:307] 4500) loss = 3.566, grad_norm = 0.218 +I0911 19:20:40.311736 139735811856128 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.220346, loss=3.50436 +I0911 19:20:40.315048 139763972338880 submission.py:307] 5000) loss = 3.504, grad_norm = 0.220 +I0911 19:24:09.280971 139735803463424 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.213338, loss=3.50955 +I0911 19:24:09.284109 139763972338880 submission.py:307] 5500) loss = 3.510, grad_norm = 0.213 +I0911 19:27:38.243821 139735811856128 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.190012, loss=3.49698 +I0911 19:27:38.247045 139763972338880 submission.py:307] 6000) loss = 3.497, grad_norm = 0.190 +I0911 19:28:39.401021 139763972338880 spec.py:333] Evaluating on the training split. +I0911 19:28:57.433778 139763972338880 spec.py:346] Evaluating on the validation split. +I0911 19:31:48.162038 139763972338880 spec.py:363] Evaluating on the test split. +I0911 19:31:48.162694 139763972338880 submission_runner.py:516] Time since start: 3031.09s, Step: 6146, {'train/loss': 3.466053009033203, 'train/ppl': 32.01014900854314, 'validation/loss': 3.458917564138427, 'validation/ppl': 31.782555311014857, 'validation/num_examples': 100000, 'test/loss': 0.0, 'test/ppl': 1.0, 'test/num_examples': 0, 'score': 2613.3255043029785, 'total_duration': 3031.0862398147583, 'accumulated_submission_time': 2613.3255043029785, 'accumulated_eval_time': 413.6452341079712, 'accumulated_logging_time': 0.04180479049682617} +I0911 19:31:48.183355 139735803463424 logging_writer.py:48] [6146] accumulated_eval_time=413.645, accumulated_logging_time=0.0418048, accumulated_submission_time=2613.33, global_step=6146, preemption_count=0, score=2613.33, test/loss=0, test/num_examples=0, test/ppl=1, total_duration=3031.09, train/loss=3.46605, train/ppl=32.0101, validation/loss=3.45892, validation/num_examples=100000, validation/ppl=31.7826 +I0911 19:34:17.024529 139735811856128 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.176083, loss=3.39211 +I0911 19:34:17.027547 139763972338880 submission.py:307] 6500) loss = 3.392, grad_norm = 0.176 +I0911 19:37:45.893343 139735803463424 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.170934, loss=3.4183 +I0911 19:37:45.896480 139763972338880 submission.py:307] 7000) loss = 3.418, grad_norm = 0.171 +I0911 19:41:14.801310 139735811856128 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.163011, loss=3.46124 +I0911 19:41:14.804433 139763972338880 submission.py:307] 7500) loss = 3.461, grad_norm = 0.163 +I0911 19:44:43.684625 139735803463424 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.162153, loss=3.4257 +I0911 19:44:43.687786 139763972338880 submission.py:307] 8000) loss = 3.426, grad_norm = 0.162 +I0911 19:48:12.599257 139735811856128 logging_writer.py:48] [8500] global_step=8500, grad_norm=0.156455, loss=3.3786 +I0911 19:48:12.602507 139763972338880 submission.py:307] 8500) loss = 3.379, grad_norm = 0.156 +I0911 19:51:41.543694 139735803463424 logging_writer.py:48] [9000] global_step=9000, grad_norm=0.156523, loss=3.36242 +I0911 19:51:41.546889 139763972338880 submission.py:307] 9000) loss = 3.362, grad_norm = 0.157 +I0911 19:55:10.339560 139735811856128 logging_writer.py:48] [9500] global_step=9500, grad_norm=0.145001, loss=3.40642 +I0911 19:55:10.342673 139763972338880 submission.py:307] 9500) loss = 3.406, grad_norm = 0.145 +I0911 19:58:39.190506 139735803463424 logging_writer.py:48] [10000] global_step=10000, grad_norm=0.138121, loss=3.32296 +I0911 19:58:39.193705 139763972338880 submission.py:307] 10000) loss = 3.323, grad_norm = 0.138 +I0911 20:02:08.042866 139735811856128 logging_writer.py:48] [10500] global_step=10500, grad_norm=0.141208, loss=3.39661 +I0911 20:02:08.046285 139763972338880 submission.py:307] 10500) loss = 3.397, grad_norm = 0.141 +I0911 20:05:36.882253 139735803463424 logging_writer.py:48] [11000] global_step=11000, grad_norm=0.136448, loss=3.27632 +I0911 20:05:36.885576 139763972338880 submission.py:307] 11000) loss = 3.276, grad_norm = 0.136 +I0911 20:09:05.731206 139735811856128 logging_writer.py:48] [11500] global_step=11500, grad_norm=0.136914, loss=3.23481 +I0911 20:09:05.734534 139763972338880 submission.py:307] 11500) loss = 3.235, grad_norm = 0.137 +I0911 20:12:34.614608 139735803463424 logging_writer.py:48] [12000] global_step=12000, grad_norm=0.13332, loss=3.31982 +I0911 20:12:34.617948 139763972338880 submission.py:307] 12000) loss = 3.320, grad_norm = 0.133 +I0911 20:14:40.063895 139763972338880 spec.py:333] Evaluating on the training split. +I0911 20:14:58.110746 139763972338880 spec.py:346] Evaluating on the validation split. +I0911 20:17:49.025152 139763972338880 spec.py:363] Evaluating on the test split. +I0911 20:17:49.025815 139763972338880 submission_runner.py:516] Time since start: 5791.95s, Step: 12300, {'train/loss': 3.2905982971191405, 'train/ppl': 26.85892846917991, 'validation/loss': 3.278693079643542, 'validation/ppl': 26.541062967911998, 'validation/num_examples': 100000, 'test/loss': 0.0, 'test/ppl': 1.0, 'test/num_examples': 0, 'score': 5181.962565422058, 'total_duration': 5791.94936466217, 'accumulated_submission_time': 5181.962565422058, 'accumulated_eval_time': 602.607102394104, 'accumulated_logging_time': 0.07037472724914551} +I0911 20:17:49.048414 139735811856128 logging_writer.py:48] [12300] accumulated_eval_time=602.607, accumulated_logging_time=0.0703747, accumulated_submission_time=5181.96, global_step=12300, preemption_count=0, score=5181.96, test/loss=0, test/num_examples=0, test/ppl=1, total_duration=5791.95, train/loss=3.2906, train/ppl=26.8589, validation/loss=3.27869, validation/num_examples=100000, validation/ppl=26.5411 +I0911 20:19:13.554959 139735803463424 logging_writer.py:48] [12500] global_step=12500, grad_norm=0.122619, loss=3.23642 +I0911 20:19:13.558076 139763972338880 submission.py:307] 12500) loss = 3.236, grad_norm = 0.123 +I0911 20:22:42.222485 139735811856128 logging_writer.py:48] [13000] global_step=13000, grad_norm=0.127325, loss=3.32034 +I0911 20:22:42.225719 139763972338880 submission.py:307] 13000) loss = 3.320, grad_norm = 0.127 +I0911 20:26:11.065748 139735803463424 logging_writer.py:48] [13500] global_step=13500, grad_norm=0.119928, loss=3.22378 +I0911 20:26:11.068928 139763972338880 submission.py:307] 13500) loss = 3.224, grad_norm = 0.120 +I0911 20:29:39.901938 139735811856128 logging_writer.py:48] [14000] global_step=14000, grad_norm=0.117869, loss=3.29569 +I0911 20:29:39.905191 139763972338880 submission.py:307] 14000) loss = 3.296, grad_norm = 0.118 +I0911 20:33:08.761705 139735803463424 logging_writer.py:48] [14500] global_step=14500, grad_norm=0.117992, loss=3.21283 +I0911 20:33:08.764969 139763972338880 submission.py:307] 14500) loss = 3.213, grad_norm = 0.118 +I0911 20:36:37.629409 139735811856128 logging_writer.py:48] [15000] global_step=15000, grad_norm=0.114852, loss=3.22927 +I0911 20:36:37.632820 139763972338880 submission.py:307] 15000) loss = 3.229, grad_norm = 0.115 +I0911 20:40:06.490961 139735803463424 logging_writer.py:48] [15500] global_step=15500, grad_norm=0.111421, loss=3.23412 +I0911 20:40:06.494086 139763972338880 submission.py:307] 15500) loss = 3.234, grad_norm = 0.111 +I0911 20:43:35.288583 139735811856128 logging_writer.py:48] [16000] global_step=16000, grad_norm=0.108328, loss=3.18099 +I0911 20:43:35.291715 139763972338880 submission.py:307] 16000) loss = 3.181, grad_norm = 0.108 +I0911 20:47:04.159064 139735803463424 logging_writer.py:48] [16500] global_step=16500, grad_norm=0.109052, loss=3.2039 +I0911 20:47:04.162192 139763972338880 submission.py:307] 16500) loss = 3.204, grad_norm = 0.109 +I0911 20:50:33.028659 139735811856128 logging_writer.py:48] [17000] global_step=17000, grad_norm=0.105648, loss=3.18207 +I0911 20:50:33.031789 139763972338880 submission.py:307] 17000) loss = 3.182, grad_norm = 0.106 +I0911 20:54:01.874623 139735803463424 logging_writer.py:48] [17500] global_step=17500, grad_norm=0.102925, loss=3.2082 +I0911 20:54:01.877755 139763972338880 submission.py:307] 17500) loss = 3.208, grad_norm = 0.103 +I0911 20:57:30.732680 139735811856128 logging_writer.py:48] [18000] global_step=18000, grad_norm=0.105047, loss=3.21049 +I0911 20:57:30.735792 139763972338880 submission.py:307] 18000) loss = 3.210, grad_norm = 0.105 +I0911 21:00:40.920991 139763972338880 spec.py:333] Evaluating on the training split. +I0911 21:00:58.952296 139763972338880 spec.py:346] Evaluating on the validation split. +I0911 21:03:49.692125 139763972338880 spec.py:363] Evaluating on the test split. +I0911 21:03:49.692757 139763972338880 submission_runner.py:516] Time since start: 8552.62s, Step: 18455, {'train/loss': 3.202191162109375, 'train/ppl': 24.58634388344877, 'validation/loss': 3.190901109934463, 'validation/ppl': 24.31032385040944, 'validation/num_examples': 100000, 'test/loss': 0.0, 'test/ppl': 1.0, 'test/num_examples': 0, 'score': 7750.686523199081, 'total_duration': 8552.616321086884, 'accumulated_submission_time': 7750.686523199081, 'accumulated_eval_time': 791.3788175582886, 'accumulated_logging_time': 0.1010887622833252} +I0911 21:03:49.714180 139735803463424 logging_writer.py:48] [18455] accumulated_eval_time=791.379, accumulated_logging_time=0.101089, accumulated_submission_time=7750.69, global_step=18455, preemption_count=0, score=7750.69, test/loss=0, test/num_examples=0, test/ppl=1, total_duration=8552.62, train/loss=3.20219, train/ppl=24.5863, validation/loss=3.1909, validation/num_examples=100000, validation/ppl=24.3103 +I0911 21:04:09.772276 139735811856128 logging_writer.py:48] [18500] global_step=18500, grad_norm=0.104617, loss=3.12964 +I0911 21:04:09.775074 139763972338880 submission.py:307] 18500) loss = 3.130, grad_norm = 0.105 +I0911 21:07:38.094745 139735803463424 logging_writer.py:48] [19000] global_step=19000, grad_norm=0.104328, loss=3.23505 +I0911 21:07:38.097871 139763972338880 submission.py:307] 19000) loss = 3.235, grad_norm = 0.104 +I0911 21:11:06.875363 139735811856128 logging_writer.py:48] [19500] global_step=19500, grad_norm=0.103594, loss=3.21332 +I0911 21:11:06.878681 139763972338880 submission.py:307] 19500) loss = 3.213, grad_norm = 0.104 +I0911 21:14:35.701996 139735803463424 logging_writer.py:48] [20000] global_step=20000, grad_norm=0.105567, loss=3.09047 +I0911 21:14:35.705133 139763972338880 submission.py:307] 20000) loss = 3.090, grad_norm = 0.106 +I0911 21:18:04.567746 139735811856128 logging_writer.py:48] [20500] global_step=20500, grad_norm=0.10024, loss=3.12662 +I0911 21:18:04.570787 139763972338880 submission.py:307] 20500) loss = 3.127, grad_norm = 0.100 +I0911 21:21:33.476871 139735803463424 logging_writer.py:48] [21000] global_step=21000, grad_norm=0.100148, loss=3.13243 +I0911 21:21:33.480005 139763972338880 submission.py:307] 21000) loss = 3.132, grad_norm = 0.100 +I0911 21:25:02.362744 139735811856128 logging_writer.py:48] [21500] global_step=21500, grad_norm=0.0955079, loss=3.17231 +I0911 21:25:02.365844 139763972338880 submission.py:307] 21500) loss = 3.172, grad_norm = 0.096 +I0911 21:28:31.224794 139735803463424 logging_writer.py:48] [22000] global_step=22000, grad_norm=0.0931489, loss=3.22211 +I0911 21:28:31.227832 139763972338880 submission.py:307] 22000) loss = 3.222, grad_norm = 0.093 +I0911 21:32:00.081736 139735811856128 logging_writer.py:48] [22500] global_step=22500, grad_norm=0.0951964, loss=3.20982 +I0911 21:32:00.084855 139763972338880 submission.py:307] 22500) loss = 3.210, grad_norm = 0.095 +I0911 21:35:28.968810 139735803463424 logging_writer.py:48] [23000] global_step=23000, grad_norm=0.0969174, loss=3.08953 +I0911 21:35:28.971735 139763972338880 submission.py:307] 23000) loss = 3.090, grad_norm = 0.097 +I0911 21:38:57.842157 139735811856128 logging_writer.py:48] [23500] global_step=23500, grad_norm=0.0898363, loss=3.16319 +I0911 21:38:57.845649 139763972338880 submission.py:307] 23500) loss = 3.163, grad_norm = 0.090 +I0911 21:42:26.653769 139735803463424 logging_writer.py:48] [24000] global_step=24000, grad_norm=0.102909, loss=3.12792 +I0911 21:42:26.656882 139763972338880 submission.py:307] 24000) loss = 3.128, grad_norm = 0.103 +I0911 21:45:55.463252 139735811856128 logging_writer.py:48] [24500] global_step=24500, grad_norm=0.0898039, loss=3.14852 +I0911 21:45:55.466315 139763972338880 submission.py:307] 24500) loss = 3.149, grad_norm = 0.090 +I0911 21:46:41.547969 139763972338880 spec.py:333] Evaluating on the training split. +I0911 21:46:59.599065 139763972338880 spec.py:346] Evaluating on the validation split. +I0911 21:49:50.500432 139763972338880 spec.py:363] Evaluating on the test split. +I0911 21:49:50.501061 139763972338880 submission_runner.py:516] Time since start: 11313.42s, Step: 24610, {'train/loss': 3.1430374145507813, 'train/ppl': 23.1741495649204, 'validation/loss': 3.130363600943095, 'validation/ppl': 22.88229805519267, 'validation/num_examples': 100000, 'test/loss': 0.0, 'test/ppl': 1.0, 'test/num_examples': 0, 'score': 10319.529506444931, 'total_duration': 11313.424616098404, 'accumulated_submission_time': 10319.529506444931, 'accumulated_eval_time': 980.3318119049072, 'accumulated_logging_time': 0.13015317916870117} +I0911 21:49:50.525649 139735803463424 logging_writer.py:48] [24610] accumulated_eval_time=980.332, accumulated_logging_time=0.130153, accumulated_submission_time=10319.5, global_step=24610, preemption_count=0, score=10319.5, test/loss=0, test/num_examples=0, test/ppl=1, total_duration=11313.4, train/loss=3.14304, train/ppl=23.1741, validation/loss=3.13036, validation/num_examples=100000, validation/ppl=22.8823 +I0911 21:52:34.261528 139735811856128 logging_writer.py:48] [25000] global_step=25000, grad_norm=0.0978632, loss=3.1133 +I0911 21:52:34.264635 139763972338880 submission.py:307] 25000) loss = 3.113, grad_norm = 0.098 +I0911 21:56:03.011642 139735803463424 logging_writer.py:48] [25500] global_step=25500, grad_norm=0.0904389, loss=3.12338 +I0911 21:56:03.014721 139763972338880 submission.py:307] 25500) loss = 3.123, grad_norm = 0.090 +I0911 21:59:31.813566 139735811856128 logging_writer.py:48] [26000] global_step=26000, grad_norm=0.0890102, loss=3.14372 +I0911 21:59:31.816641 139763972338880 submission.py:307] 26000) loss = 3.144, grad_norm = 0.089 +I0911 22:03:00.649780 139735803463424 logging_writer.py:48] [26500] global_step=26500, grad_norm=0.0920315, loss=3.08516 +I0911 22:03:00.652701 139763972338880 submission.py:307] 26500) loss = 3.085, grad_norm = 0.092 +I0911 22:06:29.451318 139735811856128 logging_writer.py:48] [27000] global_step=27000, grad_norm=0.0918943, loss=3.07421 +I0911 22:06:29.454442 139763972338880 submission.py:307] 27000) loss = 3.074, grad_norm = 0.092 +I0911 22:09:58.286328 139735803463424 logging_writer.py:48] [27500] global_step=27500, grad_norm=0.0912201, loss=3.15301 +I0911 22:09:58.289347 139763972338880 submission.py:307] 27500) loss = 3.153, grad_norm = 0.091 +I0911 22:13:27.125559 139735811856128 logging_writer.py:48] [28000] global_step=28000, grad_norm=0.0889787, loss=3.09085 +I0911 22:13:27.128843 139763972338880 submission.py:307] 28000) loss = 3.091, grad_norm = 0.089 +I0911 22:16:55.971328 139735803463424 logging_writer.py:48] [28500] global_step=28500, grad_norm=0.0930773, loss=3.10612 +I0911 22:16:55.974379 139763972338880 submission.py:307] 28500) loss = 3.106, grad_norm = 0.093 +I0911 22:20:24.866701 139735811856128 logging_writer.py:48] [29000] global_step=29000, grad_norm=0.0919089, loss=3.08057 +I0911 22:20:24.869637 139763972338880 submission.py:307] 29000) loss = 3.081, grad_norm = 0.092 +I0911 22:23:53.750704 139735803463424 logging_writer.py:48] [29500] global_step=29500, grad_norm=0.0852187, loss=3.10156 +I0911 22:23:53.753729 139763972338880 submission.py:307] 29500) loss = 3.102, grad_norm = 0.085 +I0911 22:27:22.608004 139735811856128 logging_writer.py:48] [30000] global_step=30000, grad_norm=0.0920981, loss=3.05721 +I0911 22:27:22.611037 139763972338880 submission.py:307] 30000) loss = 3.057, grad_norm = 0.092 +I0911 22:30:51.480327 139735803463424 logging_writer.py:48] [30500] global_step=30500, grad_norm=0.0921593, loss=3.08131 +I0911 22:30:51.485231 139763972338880 submission.py:307] 30500) loss = 3.081, grad_norm = 0.092 +I0911 22:32:42.295324 139763972338880 spec.py:333] Evaluating on the training split. +I0911 22:33:00.349437 139763972338880 spec.py:346] Evaluating on the validation split. +I0911 22:35:51.432903 139763972338880 spec.py:363] Evaluating on the test split. +I0911 22:35:51.433494 139763972338880 submission_runner.py:516] Time since start: 14074.36s, Step: 30765, {'train/loss': 3.0971553802490233, 'train/ppl': 22.13489627707, 'validation/loss': 3.084546959918478, 'validation/ppl': 21.85756224687316, 'validation/num_examples': 100000, 'test/loss': 0.0, 'test/ppl': 1.0, 'test/num_examples': 0, 'score': 12888.320937633514, 'total_duration': 14074.357086896896, 'accumulated_submission_time': 12888.320937633514, 'accumulated_eval_time': 1169.4699122905731, 'accumulated_logging_time': 0.16238999366760254} +I0911 22:35:51.456528 139735811856128 logging_writer.py:48] [30765] accumulated_eval_time=1169.47, accumulated_logging_time=0.16239, accumulated_submission_time=12888.3, global_step=30765, preemption_count=0, score=12888.3, test/loss=0, test/num_examples=0, test/ppl=1, total_duration=14074.4, train/loss=3.09716, train/ppl=22.1349, validation/loss=3.08455, validation/num_examples=100000, validation/ppl=21.8576 +I0911 22:35:51.993564 139735803463424 logging_writer.py:48] [30765] global_step=30765, preemption_count=0, score=12888.3 +I0911 22:35:52.001876 139763972338880 submission_runner.py:857] Final finewebedu_lm score: 12888.320937633514 diff --git a/logs/self_tuning/ademamix_golden/study_0/finewebedu_lm_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_0/finewebedu_lm_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..42b88a3d --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/finewebedu_lm_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,7 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/loss,test/num_examples,test/ppl,total_duration,train/loss,train/ppl,validation/loss,validation/num_examples,validation/ppl +224.88363099098208,0.0,45.22602105140686,1,0,45.22602105140686,0.0,0,1.0,270.7221882343292,11.000336456298829,59894.29013665645,10.9999313159367,100000,59870.02945708319 +413.6452341079712,0.0418047904968261,2613.3255043029785,6146,0,2613.3255043029785,0.0,0,1.0,3031.0862398147583,3.466053009033203,32.01014900854314,3.458917564138427,100000,31.782555311014857 +602.607102394104,0.0703747272491455,5181.962565422058,12300,0,5181.962565422058,0.0,0,1.0,5791.94936466217,3.2905982971191405,26.85892846917991,3.278693079643542,100000,26.541062967912 +791.3788175582886,0.1010887622833252,7750.686523199081,18455,0,7750.686523199081,0.0,0,1.0,8552.616321086884,3.202191162109375,24.58634388344877,3.190901109934463,100000,24.31032385040944 +980.3318119049072,0.1301531791687011,10319.529506444933,24610,0,10319.529506444933,0.0,0,1.0,11313.424616098404,3.1430374145507813,23.1741495649204,3.130363600943095,100000,22.88229805519267 +1169.4699122905731,0.16238999366760254,12888.320937633514,30765,0,12888.320937633514,0.0,0,1.0,14074.357086896896,3.0971553802490233,22.13489627707,3.084546959918478,100000,21.85756224687316 diff --git a/logs/self_tuning/ademamix_golden/study_0/finewebedu_lm_pytorch/trial_1/events.out.tfevents.1789152077.1fd2a5398af3.38.0.v2 b/logs/self_tuning/ademamix_golden/study_0/finewebedu_lm_pytorch/trial_1/events.out.tfevents.1789152077.1fd2a5398af3.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..c49a67da9e171326a925070c40b9dc8da7ab0ce4 GIT binary patch literal 39308 zcmbW=c|Z+o9|!Ov)j1U*TehSqLMlpbN=kOwHInaj(be({ z_VDp5In{=U{dSR0m0s_LqmLgTYLZd=M&u~1^a|}2bcDE zvC*VFtvL0Pqio~lWsiQPwP+%5ELZUjm^iVkTwS?_lD^8O4{f^i(vZvKY7<9$_<96S z+?75=wQYIrn%T2o$$s80uWcqbkjusfdw99|2L${54)|GCW(xsFY$r4|FAvb%sy9@o zV!53AmmA5O$u-9M26%e-x=jr64*VVQ>;DW9pAFWowJ48pHLo*LrrZBh?VA6tUt76F zbGe~h9`fs$v=KTylugl5*!}J-SjKf1lA(}r8G(Vmzn9Ug#HTwfBj!$zdL7D_u~^OW zgG_Zai7=73kXH-y@b&TX2=NKPwHz!H9n*&U0$W;@2W^}ZF-&GNfdsXb8_TN-f&>dv ze}B+za0)z`j}^dX60)#s5kVjtnnNYVJL4ZR}c9WAy6qPMRtaS!Ul;~ zI+iEsHr+8&);fwLw3eI5t1A+i9`R!2HG;0RDUb3S)o_3;Vg-rPm+Q%^O#F4l{BC&g z3yZF>k_f%#ZqO&!L{hHkQ%FF_zvr%3Xt1K`zjJ~{&bNo0MwbuYax7oQoBVNUvZ)(M zj+MN2!z8)doUTzaZJ^g$pPW<;IatrbClU`gir#)>umxm7Y4)c(8-cCZz z;}r=c)aUIA}V}5nuX3a1BHIj3{Fvr-psA`iZ&kEjlNYt@o);MW-ES(xAXC_51xS{ zbtZ|BRw&uQsj;G-O93c8v|7z6D$|Nr_98{2tghjpIM$w2vlV4xMeVZKTA;XYOCqEd zcBnWx^U-5aWUbrJDO6}hr5Ua9hP0}+fh#C>ZY9-h1$ArPyVC6qC`Rrg5z>lCRJiTU z-v^4%oo{f8Dzu`?;8v(u6J`Gt6z{r|YPNzpqw1&VD1jomKZ%f5B%wkhz04jIb1%Q+ z6sokssh}k)Ql}&bf@0-OQq5LSy_nK$P&6pAACd@ZMFuKn{yh2!6tgMHgqHCl1A zW(!pKZ+345iUwUsbp;hG-L7{B#h9KXLR#?{6{lLM;Z8d?+kjK3(+cl>MyU8;f5ih7 zp7%+01r?#Lcdmh=^a+WORw&^*?lw3P0lk>F!;VwPXoW1Y8CJXIobm<5`MspNf{LdP z8k_{hf}!gX*HR6JMfa|IN~{7E%iLG859;!@^-VoDf^kXG2?IyP;z?goku?^8L2 z7Ogmatr1qsCuFFCqVO}RW-Cg?_wJgthOP$1wlWeStr&po=$uvN1&Vp07db^$T47YR zAyyXz)ou)m(-EYat)T7~H}Z6Qf?~~F5+SW{$92SAd$14`b(`dKifXi?_*et1uD;@B zCr~^zBGqgK)r)b317?Fl-j+m2D*|yH6Cb|12#Tibl{~)2q!lr^jxElqVW9Zjcrd4^$xzIwgVhPf2BScc(417W71VWc@>1eLP_%4ABBT}Z zxDLHZK_fw-l@h=ybZCW2Y%Q#IatS&NioA4E%~pIBpHV(q&M!c5_%Ml(RwUs%o)#At zf?`g~#hjuRtq3|UQgn;_mI{h+3sTKid=o1yGyDGlg{=*VkXCF%#oH;34MFi_=N3*; zn^sIu*TL$BGxGL=;`u&OT|vchz2iBcXnLGPNGsBC9o>&!&;Uiy@Kcy$Gwf?lLIO`j7}|#X($$*294&poo6`kWbfwV)A}ta7JnlV(uxdJm^pXv3<{OuUpYlRT2cL)s25e+TCRm&baE%vY{h4B zFD|xB*966aaU?=oaUK;D25i75kf2LCx{7llsPHmtQ5$;U5oF3K>eGr7<3x%FUF!V=#hIz3nyvUE?#1-ve^NkkdNzrW zR@^~Ft4B{#K(R}s52t8AD_ZQTfz`2r*-D_8T9Z^)P!Z+kcmWiP>yrp+#baDY`{=rh zK+$~eSWeN9R_saUJ53THttiBGjEdU52^2Q#*K+G?R|;3$cZR;Zw2`S6j=K+(wT1EZl0(v3xNo)*sibuej?qr4{GWt6?wRRT_w&h+jBMs@V$a-rX$!v;`<$Um+3F z3QbhJnX*3)6h~c*IE4YN*f_Q-DmIPWR|>sQ9Zjm)3hIpV%6ohf6j#TS2x)~jD$<|S zP6Wl5JbO-INGr?-i4+>!j7NZC${SM6R!}?bAl8>)ocaTi}^OHL;%Rx}^|$F={c1kXGoSVsd)fDNuAO+RZ79X@&YUk)rB{b@xHh<{PPI zE2uq6`P=x-pa`v;{(Aw^3IkMhzZ`*k)S=+ZTu2LAk-1T%cs*~(U{G8NC)I2PbzQs+ zPv{4Vm9t2Mw897#s%1a$Z-d_}z2X!tX+`a1O}x5u9cTS48sLs!S(r!f8ONrbe*1{DKd=R5#K!k%DGVM;3ob`mM{Pirj% zh3hd=%~nu*)RW;`eV`YIGD(EA!VVQmUzHYs;;d;jr)Wzn<~@+(b#bxVOLtHVw<6UQ zRE)f=odF74I}#zSa6m;#_UeA1ICm_WQ<%|;Yq}!Ero#9$pg5LEsw=3F`K+=5h4mE@ zA+6|viq~Bh90Wy@MJA^(rxit?MT%?DW4D4L!iH3{71VW+*=tyRP=q*;2x&zhR1{rW z(FPPh*FNSH7PLZhjSSBy@9ZxBfFf}Vsb(vvCy>E${ab?~D3wG=D+Zt+aRoo`9-2t1 z*$V2D;>J@OIf24v6N!*kxTB)Y2mbQ*@ve|JaGHZUd)rm!TKUzmjUUf_efe z{d%GuC=Mx&|GfZdg%>Io9(#-5e!S#4mkY6`6~Q*5Ui@&CT>?e6FR5lLs6EQPFcm-7 z`xrtZq!r^(kt}aH0eTU8Es0Zfq!ojQimr?6hpHrk;`AL-%~nu*l!aduKADFJwu0Je%f83qH?+HaBoWez5LDb7|7|z)Vz3?M5@^e>gt|xWbsny#m#UMA+4BM!&-0(J6aKxC{j2^ zue%EhCl6B1R#1OcbJDX30)?^b}_A182SBdZfC7iVsu2+yzBJ6A~e< zn1_lG75N!ZxG#<86b`ha^FtN9y3YpNJcM4T#gS^Zg1YN%k50Y^3i&z`A+1<|iprz+ zodiW*^SzwHkyg|V5h=#E%0CB+t8GX%TR~kHZ9azL)m>^yBBT|IP!SU{b0#RN9=XaX zy3q>X2$4b_SrrxLnWUPnpgv=5+F@FI=*5W3Btlvdg^K&fSK~A7!_IFwMR!_Jb6sUT zqx1%TFo0gD_8`@41=S0$3xn`)=DGbzgtQ_W6-`pt;-@gy7gZZ8?s`3Fh26+XsPHT6 zgFEfm98%3zP}fD@NsVlw7oHDDgtQ_C6*rx;w!s+{KS7^U^rRJuqeQ*P7p- zRx;>CdY&?!}~tN zK(RC`hEw#V6}Nqr@#@}a8P^IFBUX@Vwu0KD41bPj1Pawe5+SWfK*hGcI)^~xAL9s=XRI?S-H?;pzZi3(W->gF-q!o#%C?06k6BH9>pW_t$X+`uM zkwV??KK|N1crmGFE2yhm&q*@@dSSerL`W;vq9P*22)}z^Wb~9%44@TX!bOT|=5vdn z7fnq`HCsW=7Xiil@w*4NEJ=j4A_)~rsqLpjFO*V#a*BbpB7VL|@nOqIPf*yVlWMkt zx-O3PG^hf-7<-sRNGmp=Vq^8pC{VOB($!PkyPar-eY{9sq(F!5P(HI*E{0>_COGL#{3;0;+A|6vJpm zq(>R{V(pB)QJ}a|hg4Tkku`5j0Vp;$CK1w#R8*J_bjF?5df5q1F`QO(>m|CnFU{4_ z0!4=eQq5LSJMH(Ud51vpY(0sPR-~a~a?f4|pcidr_c?_#t=N0~JNBZ;WbS5Am{cd# z6;v!#T7DE1y7fqev?3i9N0NQx>y=yd4-1SD#3Z;GDuoow;zr?pUR2f97*$V2({N|A-3qVoe zLL#IU`%!U4zf~zHT3u1{^(Fo?dgntDA+0!w ziZhRv;JaCx4eH4$M$w9-?9ZrJcvDXk&Zy5Wq?)auu5Qctlu6Kwcb+6dT5%W^b#l|| zKra@5^Wqe)v|_@IFQ_EC2x&zIDmp|q?GB3R+YfLG4_Yz4c?q6ThYy)g z1jVvFq?)auu8VUk?sK|Jei%;f*E*5Z#(X_&TwrGzEXm)oF^kV1@Qq5LSXH>1v zg^r-GxJM$S6=zX#{zlRW=tbBBxxV7A=S3^jI*HDxg*`WH1;xA&Qq5LSXVeQ7?K+_7 zJDWsEE6$_hP08ll@Qyv1RYq@xqL}eAuN&Tb@WL2A|FQ7)_+*kd)c|3e1VWc>1yW0EsG5iBv#dGTy$V(qtJzO5?i!1jtgQDx{nZ zc`k;Wu9tfcAV<~t3~5NX1H4_WV^&~4fHe50*IotW0poQ!YER}OxWdY=&JD(v5QTm3Ef`^Az0A$OW>;^i&USN8L@j?nm*D!=cTMxbt5Us6z zIU90IX){8GGSGA!{F>w*koU>U@SIB(ftybUVNA2oS#ugXStAI~YItE(&x<$k|T&ECI6T z24BvG6x0esi1qq6F94E$kI#^VWVj%t#~7VdfaIQBaYzA4V*Ez8(9{bdxII9IYU}9N`gO4{VEhVhOn1?#wXVPmcdc2v zd^sE9wZQ`+?pf1T0c55=pCJjcOGJpZg-(Bf)UNF`Q~@bv{P-$3Q3UChXt@(QGOICP z&W1RIx*%kuR=5U07B=TIBq5KtBgFh_`cZ&r99X(Y0V!nsT4v%EH@rr!?w;QiAQvW`08Iq95BM1rJJajueur}%PhkaIV5-)-Qu z$m2s49O_lr-f|^;==&kRt_!c_HGauw{OMiTFI`2esg`g6H$C#F9t^9P&P4=!DpGChByWNnccmf8lR`!5y9jeXO z|KXL`9zmTjsJGsb)!oV$65Q_4W|@6MKB%Ja#HNwv z?#2}y2|iw4>ug`1w|lN(l59dtKCeP=#g6RQvJ*T(HS1DZ1vY})W_+aNMxdR6hwsGj z__6Sg^t29q%HO>g+kO8Xd^U|s=~AHBQ=Tz?GBv2KGwvpbwT;u^BGK5(m$NrV>;4wF zIeM8m2f;CUcZAQ7JSKWscuZm|uTg?szGwF-`n7-EckeKM<1!-C1tI(IPv`*=Gbez|@a4J|k}qdN z4z<(6jy!MItrQ?F7w{R9kZUU4VE2hO@R$dXj2E6C6_9O=?`b}2I}9PN`@FmX(%}PN z&W0@RE`pfbmBaxg_Y0pP2}#mKh`U|&$pHCzb7k{7zb^Jf#uw|(RuWz8E2E3@0AiQR zm$M;_r|MuwY>Xf10_1x>pCJhe)Imrc%R!d`vcJ!z5ei5Q`FpX_sJN2asW|e1;^%t^q=#Q(pN4~GbJkFRq%E^Lqkw#2d}E@=u7TJQZB6-7fOJ~J zm$M-S2Q?6~X#T`C08!q;XGlUGw?s&T@hv6+CV&i^ z#+S1pjpnK&B(bg0B7mHm!)HiBB0D0az+nD6nBQJl*iThJqTHqCw-&3MurJ4(Mr8sg zx-%c5;862h?c7^9EcIT|>kiCsYr63nf12Ow-x8hf{%07|EgpPE#pXAA-=_0metR~K zkNC^^ZL3L}`Y^wp4B)fq`EAep_4pvuO27Rz=$85F!;iv6 za&aqP&fXl~H2#jqM7ijpJ{*&qyZ8*rV=}-VkBMP?m2Uu1caA=zfSmWHKMxIw6z%dV z2aXH}NH0&moDE6vsD!(GM4h%V0NFjB&ya-Z^+rgj@$4diG#YT`ivn`ghlWgW>WBNg zaZbxI0J-PPm$M-k5zjB<8K? zM)kgUS1o`j=kn!jNPX`Q2pK%-5|ZJ9kS7=V9RkSg2c{kh$fh6~QfXaZ z?8v~tTbBVcE1xfCLzZk7LGmsRDF8^d0zN|$661-GPm?#`Lw54maVr&&_)r?sZ-?mA zDh*fb2aqp;d^sC3*zyf_#G&d2HGm{c<})NA?&A?M_Nrf1=!o+Cbr%$ns3|n$>UR-D zb7@iaayI0W=`(~3vluuNAZqb^h9tx^7$Ito2CD#4clnJU3P@xG4e?#s8&9oX z9g6T1(b0)~IU92S-a~{G4~dxzknD|oh9pEO93eYi+*t^Z7V@5QGu>YodvT;PE!m|h zx=5TG?AQy>w#S?V^=!(qp}9yozcf+HviuLOFJ4}BmUVRzgh*OmMxrHiiQ#y#>?!Or zMqzn5hheF^Rn(Qali%X2Zr^?t)Uz$EUGJd9I&en-Shgv5{FX z*Q)-n>m@Em@;^(vr?3xR|H?ftmq2%V-W8-MMAS5QGwlqHP9H<-u7;<{(p*8vpT@aE zmZJ6S-EQ&`2E*_9f~Y^tbBm{*!bMqh-E0ZZ-n-rjlK!uOZrIEt_#5wc9|e)z!TQgM z?qx`S6S#F8{wfHm*hqIIPu00o`8MU=^TS(NLnVK{_kTIlJ^1qC0sI%1cc}`(=%H?U zyCR&$I=UG)hMwhV3IhHx)lJ$^1<&c0Ps?(;m%9$1{=QoFwT2+6qGKJkz(uM7h49}g z4yz}K{qMQ1^BqT=>qhOr*3Y4Qo%YVH6J&mRg20Lmc4}8~Td?0%{Ss^om5cjZZXt{dZv;u;@JP*`@vGb|0aif&_X;>PuW zzHBiP)Uz!gjB?Pjw9ko3V0mF72$8f5k4MYX*i)%s=_h-4MPW%wWLWxLbi{M)x@!6> zu(YfpsApS7nu{Kl5@&931j~+kf)GiIT_Re%woJ4FOH!agm^|0 zv~aM@og%1bTdw!IjD4}tOIr?>bCH4&NsCbuTGlOhP6NxBBv(s?Wx*zfW&aeBWz#mt zU0_+BEU0H&Jat8uY8r#yfyI5dAVkuly$LN(52&vI%dvrR6BL%xtqhBWn&@$Sz{)44 zU@34B)Uz#HR-MMaYURr>Cpz z089KlK|R}&d*(P=jxU|_0xa*B2tp(+kGG@6DCJ~VuqX|ylB2LZ+r_Y~^Aqi3Mx91K z081}-K|R|tq~sJ@-uCVM6f9%L3PL0;=XarH-=evtVCh)TR=we`YyZw3hUMB}(LUDY zQBV8@ptioCo^9DM?JQb+Mz_-iORTXVMADMB2Q7_b9YVljSUJo_VYzgGVMz<>g?qAt z@r*Oj7o)0zdbXu@=>fDnX`o~bmiRh?5J^k?0krg}p*I686N{1q6_$)649mqfqJ7MK zWS90LR6UxcJ3@(5Z!e=&*&OK!%)O$y826AX*%%%0en zS1En1z;f_{pq_1M{6^H5yF1Q?f~D(qL5QSf+zGVI&~KX$<7?mJ>vI*B4H=T->%8V7 zk=iu(CLq#3CrD9kv+ggI+Y!fbsSFWkJZF#@D=IqO;|N=`ceWh34c4qW&M*|cQWwPj_xNg4D-_4q`ahE!!uT3nO%Pb2@%3@HD)`Zr zVRR9WX&$u&nUdRrLk1o~jRp2IVSHWQP!RKXV7VJ=>ybbqFmdR}LEimbR}2A(9rWb7(1=ZBqp-A6ga%Dl9Qs z49iY4(WXB;^X4?Lgtrsavn}6x?MKV*Y{PzFd1NC9k+kS#p=Hb3E7QSpG}U;s!ZPP7 z!(u&A^w?KTY{}c*Nc35mOEHHECnHwmW+F7X=Uq(``9jrpP>p% z&Lf5;{$e*g$FjA)c!K3>UqLE!!TUrIAZ?6FA4*V=Pk? zmh(>-mPSURzIbKj;A*eZJ#~xs*zg`d`X^DA)7KeHETfx%RFz}JW zax|Y|sT3*d%dCdlwV*FuS_|sg7QOkYc)cjinw|!hX6*$bl9tK&Xc^~_o(`7wDqCyl z{krzkUNbC1lkM>w8`R_BBe3LX3hLRG;x(eaXf3r|36}9Xf)GiI`)jle9llczmY7eE zx+yH13K*957NT>^yw&p-V9_rV)Uz$J<6=vz>?p82QO)HSA!+GRfEMqlOD$mJvvhhG ztFY`WlpOi8EJbHp;rx93<8|l=L5e~|jeHI|yK%b8yKb@vM!rg(f{;Ird{#c9BW!#) z<_3%puHytze;E0CJxIkxpja6B zwucKsDmL<6k2|dkBVW`^LB?N>d|x`ZGlP*YcfKHu9{GBD^u#l9pYIrb=vl{TLBJnI zKL1xEFiB_s<~A_$y&1 | tee -a /logs/imagenet_resnet_pytorch_09-12-2026-02-32-07.log +W0912 02:32:09.324000 9 site-packages/torch/distributed/run.py:803] +W0912 02:32:09.324000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0912 02:32:09.324000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0912 02:32:09.324000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-12 02:32:10.931606: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-12 02:32:10.931605: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-12 02:32:10.931608: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-12 02:32:10.931631: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789180330.952607 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789180330.952610 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789180330.952607 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789180330.952978 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789180330.959503 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789180330.959503 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789180330.959531 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789180330.960099 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789180330.976578 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789180330.976581 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789180330.976604 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789180330.976606 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789180330.976606 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789180330.976608 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789180330.976592 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789180330.976608 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789180330.976611 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789180330.976612 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789180330.976614 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789180330.976616 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789180330.976962 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789180330.976989 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789180330.976991 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789180330.976993 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789180336.533657 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789180336.628509 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789180336.720993 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789180336.823714 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W912 02:32:20.667773719 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0912 02:32:21.088945 140668420674752 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/imagenet_resnet_pytorch. +I0912 02:32:21.088946 140529412666560 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/imagenet_resnet_pytorch. +I0912 02:32:21.088951 140411597268160 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/imagenet_resnet_pytorch. +I0912 02:32:21.088978 139848747422912 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/imagenet_resnet_pytorch. +I0912 02:32:21.112514 140411597268160 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/imagenet_resnet_pytorch/trial_1. +I0912 02:32:21.417241 140411597268160 submission_runner.py:242] Initializing dataset. +I0912 02:32:54.753580 140411597268160 submission_runner.py:251] Initializing model. +I0912 02:32:55.209222 140411597268160 submission_runner.py:290] Performing `torch.compile`. +I0912 02:32:56.262224 140411597268160 submission_runner.py:294] Initializing optimizer. +I0912 02:32:56.263072 140411597268160 submission_runner.py:299] Initializing metrics bundle. +I0912 02:32:56.263222 140411597268160 submission_runner.py:321] Initializing checkpoint and logger. +I0912 02:32:56.263715 140411597268160 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_0/imagenet_resnet_pytorch/trial_1/meta_data_0.json. +I0912 02:32:56.491293 140411597268160 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_0/imagenet_resnet_pytorch/trial_1/flags_0.json. +I0912 02:32:56.526838 140411597268160 submission_runner.py:359] Starting training loop. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +[rank0]:W0912 02:33:25.222000 38 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank3]:W0912 02:33:25.643000 41 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank2]:W0912 02:33:26.682000 40 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank1]:W0912 02:33:27.171000 39 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +/usr/local/lib/python3.11/site-packages/torch/autograd/graph.py:841: UserWarning: Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 512, 512] +bucket_view.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 1, 1] (Triggered internally at /pytorch/torch/csrc/distributed/c10d/reducer.cpp:334.) + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +/usr/local/lib/python3.11/site-packages/torch/autograd/graph.py:841: UserWarning: Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 512, 512] +bucket_view.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 1, 1] (Triggered internally at /pytorch/torch/csrc/distributed/c10d/reducer.cpp:334.) + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +/usr/local/lib/python3.11/site-packages/torch/autograd/graph.py:841: UserWarning: Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 512, 512] +bucket_view.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 1, 1] (Triggered internally at /pytorch/torch/csrc/distributed/c10d/reducer.cpp:334.) + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +/usr/local/lib/python3.11/site-packages/torch/autograd/graph.py:841: UserWarning: Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 512, 512] +bucket_view.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 1, 1] (Triggered internally at /pytorch/torch/csrc/distributed/c10d/reducer.cpp:334.) + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +I0912 02:34:58.034810 140387592111872 logging_writer.py:48] [0] global_step=0, grad_norm=0.499999, loss=6.92814 +I0912 02:34:58.071020 140411597268160 submission.py:307] 0) loss = 6.928, grad_norm = 0.500 +I0912 02:34:58.724097 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +I0912 02:36:18.575976 140411597268160 spec.py:346] Evaluating on the validation split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 02:37:55.627950 140411597268160 spec.py:363] Evaluating on the test split. +I0912 02:37:56.073336 140411597268160 dataset_info.py:707] Load dataset info from /data/imagenet/pytorch/imagenet_v2/matched-frequency/3.0.0 +I0912 02:37:56.116652 140411597268160 reader.py:262] Creating a tf.data.Dataset reading 16 files located in folders: /data/imagenet/pytorch/imagenet_v2/matched-frequency/3.0.0. +I0912 02:37:56.182181 140411597268160 logging_logger.py:49] Constructing tf.data.Dataset imagenet_v2 for split test, from /data/imagenet/pytorch/imagenet_v2/matched-frequency/3.0.0 +I0912 02:38:21.345586 140411597268160 submission_runner.py:516] Time since start: 324.82s, Step: 1, {'train/accuracy': 0.0011957908163265306, 'train/loss': 6.911289137236926, 'validation/accuracy': 0.00114, 'validation/loss': 6.91032625, 'validation/num_examples': 50000, 'test/accuracy': 0.0016, 'test/loss': 6.91218828125, 'test/num_examples': 10000, 'score': 121.5451877117157, 'total_duration': 324.81873393058777, 'accumulated_submission_time': 121.5451877117157, 'accumulated_eval_time': 202.62144112586975, 'accumulated_logging_time': 0} +I0912 02:38:21.366865 140365747074816 logging_writer.py:48] [1] accumulated_eval_time=202.621, accumulated_logging_time=0, accumulated_submission_time=121.545, global_step=1, preemption_count=0, score=121.545, test/accuracy=0.0016, test/loss=6.91219, test/num_examples=10000, total_duration=324.819, train/accuracy=0.00119579, train/loss=6.91129, validation/accuracy=0.00114, validation/loss=6.91033, validation/num_examples=50000 +I0912 02:38:22.555369 140365738682112 logging_writer.py:48] [1] global_step=1, grad_norm=0.499999, loss=6.93104 +I0912 02:38:22.558347 140411597268160 submission.py:307] 1) loss = 6.931, grad_norm = 0.500 +I0912 02:38:22.817467 140365747074816 logging_writer.py:48] [2] global_step=2, grad_norm=0.499999, loss=6.92149 +I0912 02:38:22.820424 140411597268160 submission.py:307] 2) loss = 6.921, grad_norm = 0.500 +I0912 02:38:23.079396 140365738682112 logging_writer.py:48] [3] global_step=3, grad_norm=0.499999, loss=6.93266 +I0912 02:38:23.082298 140411597268160 submission.py:307] 3) loss = 6.933, grad_norm = 0.500 +I0912 02:38:23.340991 140365747074816 logging_writer.py:48] [4] global_step=4, grad_norm=0.499999, loss=6.92965 +I0912 02:38:23.344075 140411597268160 submission.py:307] 4) loss = 6.930, grad_norm = 0.500 +I0912 02:38:23.604111 140365738682112 logging_writer.py:48] [5] global_step=5, grad_norm=0.499999, loss=6.92256 +I0912 02:38:23.607084 140411597268160 submission.py:307] 5) loss = 6.923, grad_norm = 0.500 +I0912 02:38:23.867379 140365747074816 logging_writer.py:48] [6] global_step=6, grad_norm=0.499999, loss=6.93221 +I0912 02:38:23.870442 140411597268160 submission.py:307] 6) loss = 6.932, grad_norm = 0.500 +I0912 02:38:24.131381 140365738682112 logging_writer.py:48] [7] global_step=7, grad_norm=0.499999, loss=6.9191 +I0912 02:38:24.134384 140411597268160 submission.py:307] 7) loss = 6.919, grad_norm = 0.500 +I0912 02:38:24.394356 140365747074816 logging_writer.py:48] [8] global_step=8, grad_norm=0.499999, loss=6.92509 +I0912 02:38:24.397657 140411597268160 submission.py:307] 8) loss = 6.925, grad_norm = 0.500 +I0912 02:38:24.656562 140365738682112 logging_writer.py:48] [9] global_step=9, grad_norm=0.499999, loss=6.91818 +I0912 02:38:24.659570 140411597268160 submission.py:307] 9) loss = 6.918, grad_norm = 0.500 +I0912 02:38:24.918540 140365747074816 logging_writer.py:48] [10] global_step=10, grad_norm=0.499999, loss=6.92987 +I0912 02:38:24.921458 140411597268160 submission.py:307] 10) loss = 6.930, grad_norm = 0.500 +I0912 02:38:25.180310 140365738682112 logging_writer.py:48] [11] global_step=11, grad_norm=0.499999, loss=6.92611 +I0912 02:38:25.183328 140411597268160 submission.py:307] 11) loss = 6.926, grad_norm = 0.500 +I0912 02:38:25.442640 140365747074816 logging_writer.py:48] [12] global_step=12, grad_norm=0.499999, loss=6.92024 +I0912 02:38:25.445622 140411597268160 submission.py:307] 12) loss = 6.920, grad_norm = 0.500 +I0912 02:38:25.704308 140365738682112 logging_writer.py:48] [13] global_step=13, grad_norm=0.499999, loss=6.92541 +I0912 02:38:25.707240 140411597268160 submission.py:307] 13) loss = 6.925, grad_norm = 0.500 +I0912 02:38:25.965911 140365747074816 logging_writer.py:48] [14] global_step=14, grad_norm=0.499999, loss=6.92676 +I0912 02:38:25.968998 140411597268160 submission.py:307] 14) loss = 6.927, grad_norm = 0.500 +I0912 02:38:26.227665 140365738682112 logging_writer.py:48] [15] global_step=15, grad_norm=0.499999, loss=6.92614 +I0912 02:38:26.230667 140411597268160 submission.py:307] 15) loss = 6.926, grad_norm = 0.500 +I0912 02:38:26.489265 140365747074816 logging_writer.py:48] [16] global_step=16, grad_norm=0.499999, loss=6.93205 +I0912 02:38:26.492265 140411597268160 submission.py:307] 16) loss = 6.932, grad_norm = 0.500 +I0912 02:38:26.751127 140365738682112 logging_writer.py:48] [17] global_step=17, grad_norm=0.499999, loss=6.93207 +I0912 02:38:26.754011 140411597268160 submission.py:307] 17) loss = 6.932, grad_norm = 0.500 +I0912 02:38:27.013071 140365747074816 logging_writer.py:48] [18] global_step=18, grad_norm=0.499999, loss=6.92948 +I0912 02:38:27.016137 140411597268160 submission.py:307] 18) loss = 6.929, grad_norm = 0.500 +I0912 02:38:27.275420 140365738682112 logging_writer.py:48] [19] global_step=19, grad_norm=0.499999, loss=6.92564 +I0912 02:38:27.278362 140411597268160 submission.py:307] 19) loss = 6.926, grad_norm = 0.500 +I0912 02:38:27.537663 140365747074816 logging_writer.py:48] [20] global_step=20, grad_norm=0.499999, loss=6.93512 +I0912 02:38:27.540674 140411597268160 submission.py:307] 20) loss = 6.935, grad_norm = 0.500 +I0912 02:38:27.799605 140365738682112 logging_writer.py:48] [21] global_step=21, grad_norm=0.499999, loss=6.91925 +I0912 02:38:27.802658 140411597268160 submission.py:307] 21) loss = 6.919, grad_norm = 0.500 +I0912 02:38:28.061448 140365747074816 logging_writer.py:48] [22] global_step=22, grad_norm=0.499999, loss=6.92099 +I0912 02:38:28.064429 140411597268160 submission.py:307] 22) loss = 6.921, grad_norm = 0.500 +I0912 02:38:28.323855 140365738682112 logging_writer.py:48] [23] global_step=23, grad_norm=0.499999, loss=6.91962 +I0912 02:38:28.326852 140411597268160 submission.py:307] 23) loss = 6.920, grad_norm = 0.500 +I0912 02:38:28.585771 140365747074816 logging_writer.py:48] [24] global_step=24, grad_norm=0.499999, loss=6.91426 +I0912 02:38:28.588707 140411597268160 submission.py:307] 24) loss = 6.914, grad_norm = 0.500 +I0912 02:38:28.847956 140365738682112 logging_writer.py:48] [25] global_step=25, grad_norm=0.499999, loss=6.93002 +I0912 02:38:28.851093 140411597268160 submission.py:307] 25) loss = 6.930, grad_norm = 0.500 +I0912 02:38:29.109904 140365747074816 logging_writer.py:48] [26] global_step=26, grad_norm=0.499999, loss=6.93594 +I0912 02:38:29.113031 140411597268160 submission.py:307] 26) loss = 6.936, grad_norm = 0.500 +I0912 02:38:29.371826 140365738682112 logging_writer.py:48] [27] global_step=27, grad_norm=0.499999, loss=6.93353 +I0912 02:38:29.374820 140411597268160 submission.py:307] 27) loss = 6.934, grad_norm = 0.500 +I0912 02:38:29.633743 140365747074816 logging_writer.py:48] [28] global_step=28, grad_norm=0.499999, loss=6.91684 +I0912 02:38:29.636802 140411597268160 submission.py:307] 28) loss = 6.917, grad_norm = 0.500 +I0912 02:38:29.896265 140365738682112 logging_writer.py:48] [29] global_step=29, grad_norm=0.499999, loss=6.92017 +I0912 02:38:29.899287 140411597268160 submission.py:307] 29) loss = 6.920, grad_norm = 0.500 +I0912 02:38:30.159683 140365747074816 logging_writer.py:48] [30] global_step=30, grad_norm=0.499999, loss=6.93183 +I0912 02:38:30.162990 140411597268160 submission.py:307] 30) loss = 6.932, grad_norm = 0.500 +I0912 02:38:30.422250 140365738682112 logging_writer.py:48] [31] global_step=31, grad_norm=0.499999, loss=6.9318 +I0912 02:38:30.425200 140411597268160 submission.py:307] 31) loss = 6.932, grad_norm = 0.500 +I0912 02:38:30.684316 140365747074816 logging_writer.py:48] [32] global_step=32, grad_norm=0.499999, loss=6.93248 +I0912 02:38:30.687351 140411597268160 submission.py:307] 32) loss = 6.932, grad_norm = 0.500 +I0912 02:38:30.946295 140365738682112 logging_writer.py:48] [33] global_step=33, grad_norm=0.499999, loss=6.92219 +I0912 02:38:30.949324 140411597268160 submission.py:307] 33) loss = 6.922, grad_norm = 0.500 +I0912 02:38:31.208665 140365747074816 logging_writer.py:48] [34] global_step=34, grad_norm=0.499999, loss=6.92312 +I0912 02:38:31.211554 140411597268160 submission.py:307] 34) loss = 6.923, grad_norm = 0.500 +I0912 02:38:31.470623 140365738682112 logging_writer.py:48] [35] global_step=35, grad_norm=0.499999, loss=6.92905 +I0912 02:38:31.473595 140411597268160 submission.py:307] 35) loss = 6.929, grad_norm = 0.500 +I0912 02:38:31.732302 140365747074816 logging_writer.py:48] [36] global_step=36, grad_norm=0.499999, loss=6.93064 +I0912 02:38:31.735313 140411597268160 submission.py:307] 36) loss = 6.931, grad_norm = 0.500 +I0912 02:38:31.993920 140365738682112 logging_writer.py:48] [37] global_step=37, grad_norm=0.499999, loss=6.90401 +I0912 02:38:31.996848 140411597268160 submission.py:307] 37) loss = 6.904, grad_norm = 0.500 +I0912 02:38:32.256559 140365747074816 logging_writer.py:48] [38] global_step=38, grad_norm=0.499999, loss=6.92166 +I0912 02:38:32.259405 140411597268160 submission.py:307] 38) loss = 6.922, grad_norm = 0.500 +I0912 02:38:32.518979 140365738682112 logging_writer.py:48] [39] global_step=39, grad_norm=0.499999, loss=6.92987 +I0912 02:38:32.521927 140411597268160 submission.py:307] 39) loss = 6.930, grad_norm = 0.500 +I0912 02:38:32.781454 140365747074816 logging_writer.py:48] [40] global_step=40, grad_norm=0.499999, loss=6.91581 +I0912 02:38:32.784436 140411597268160 submission.py:307] 40) loss = 6.916, grad_norm = 0.500 +I0912 02:38:33.043664 140365738682112 logging_writer.py:48] [41] global_step=41, grad_norm=0.499999, loss=6.91545 +I0912 02:38:33.046778 140411597268160 submission.py:307] 41) loss = 6.915, grad_norm = 0.500 +I0912 02:38:33.306539 140365747074816 logging_writer.py:48] [42] global_step=42, grad_norm=0.499999, loss=6.91909 +I0912 02:38:33.309576 140411597268160 submission.py:307] 42) loss = 6.919, grad_norm = 0.500 +I0912 02:38:33.568822 140365738682112 logging_writer.py:48] [43] global_step=43, grad_norm=0.499999, loss=6.92009 +I0912 02:38:33.571876 140411597268160 submission.py:307] 43) loss = 6.920, grad_norm = 0.500 +I0912 02:38:33.832051 140365747074816 logging_writer.py:48] [44] global_step=44, grad_norm=0.499999, loss=6.9202 +I0912 02:38:33.835126 140411597268160 submission.py:307] 44) loss = 6.920, grad_norm = 0.500 +I0912 02:38:34.094856 140365738682112 logging_writer.py:48] [45] global_step=45, grad_norm=0.499999, loss=6.91003 +I0912 02:38:34.097841 140411597268160 submission.py:307] 45) loss = 6.910, grad_norm = 0.500 +I0912 02:38:34.357120 140365747074816 logging_writer.py:48] [46] global_step=46, grad_norm=0.499999, loss=6.91529 +I0912 02:38:34.360122 140411597268160 submission.py:307] 46) loss = 6.915, grad_norm = 0.500 +I0912 02:38:34.619514 140365738682112 logging_writer.py:48] [47] global_step=47, grad_norm=0.499999, loss=6.91196 +I0912 02:38:34.622498 140411597268160 submission.py:307] 47) loss = 6.912, grad_norm = 0.500 +I0912 02:38:34.882484 140365747074816 logging_writer.py:48] [48] global_step=48, grad_norm=0.499999, loss=6.93097 +I0912 02:38:34.885418 140411597268160 submission.py:307] 48) loss = 6.931, grad_norm = 0.500 +I0912 02:38:35.145526 140365738682112 logging_writer.py:48] [49] global_step=49, grad_norm=0.499999, loss=6.91686 +I0912 02:38:35.148556 140411597268160 submission.py:307] 49) loss = 6.917, grad_norm = 0.500 +I0912 02:38:35.407958 140365747074816 logging_writer.py:48] [50] global_step=50, grad_norm=0.499999, loss=6.92076 +I0912 02:38:35.410942 140411597268160 submission.py:307] 50) loss = 6.921, grad_norm = 0.500 +I0912 02:38:35.670437 140365738682112 logging_writer.py:48] [51] global_step=51, grad_norm=0.499999, loss=6.92678 +I0912 02:38:35.673431 140411597268160 submission.py:307] 51) loss = 6.927, grad_norm = 0.500 +I0912 02:38:35.932661 140365747074816 logging_writer.py:48] [52] global_step=52, grad_norm=0.499999, loss=6.92405 +I0912 02:38:35.935594 140411597268160 submission.py:307] 52) loss = 6.924, grad_norm = 0.500 +I0912 02:38:36.195304 140365738682112 logging_writer.py:48] [53] global_step=53, grad_norm=0.499999, loss=6.91638 +I0912 02:38:36.198312 140411597268160 submission.py:307] 53) loss = 6.916, grad_norm = 0.500 +I0912 02:38:36.457595 140365747074816 logging_writer.py:48] [54] global_step=54, grad_norm=0.499999, loss=6.91635 +I0912 02:38:36.460558 140411597268160 submission.py:307] 54) loss = 6.916, grad_norm = 0.500 +I0912 02:38:36.720117 140365738682112 logging_writer.py:48] [55] global_step=55, grad_norm=0.499999, loss=6.91303 +I0912 02:38:36.723048 140411597268160 submission.py:307] 55) loss = 6.913, grad_norm = 0.500 +I0912 02:38:36.983814 140365747074816 logging_writer.py:48] [56] global_step=56, grad_norm=0.499999, loss=6.90619 +I0912 02:38:36.986791 140411597268160 submission.py:307] 56) loss = 6.906, grad_norm = 0.500 +I0912 02:38:37.246570 140365738682112 logging_writer.py:48] [57] global_step=57, grad_norm=0.499999, loss=6.92586 +I0912 02:38:37.249859 140411597268160 submission.py:307] 57) loss = 6.926, grad_norm = 0.500 +I0912 02:38:37.510348 140365747074816 logging_writer.py:48] [58] global_step=58, grad_norm=0.499999, loss=6.9206 +I0912 02:38:37.513278 140411597268160 submission.py:307] 58) loss = 6.921, grad_norm = 0.500 +I0912 02:38:37.773495 140365738682112 logging_writer.py:48] [59] global_step=59, grad_norm=0.499999, loss=6.91145 +I0912 02:38:37.776460 140411597268160 submission.py:307] 59) loss = 6.911, grad_norm = 0.500 +I0912 02:38:38.036027 140365747074816 logging_writer.py:48] [60] global_step=60, grad_norm=0.499999, loss=6.91159 +I0912 02:38:38.039075 140411597268160 submission.py:307] 60) loss = 6.912, grad_norm = 0.500 +I0912 02:38:38.298507 140365738682112 logging_writer.py:48] [61] global_step=61, grad_norm=0.499999, loss=6.92187 +I0912 02:38:38.301882 140411597268160 submission.py:307] 61) loss = 6.922, grad_norm = 0.500 +I0912 02:38:38.561384 140365747074816 logging_writer.py:48] [62] global_step=62, grad_norm=0.499999, loss=6.92803 +I0912 02:38:38.564410 140411597268160 submission.py:307] 62) loss = 6.928, grad_norm = 0.500 +I0912 02:38:38.824300 140365738682112 logging_writer.py:48] [63] global_step=63, grad_norm=0.499999, loss=6.91927 +I0912 02:38:38.827292 140411597268160 submission.py:307] 63) loss = 6.919, grad_norm = 0.500 +I0912 02:38:39.087221 140365747074816 logging_writer.py:48] [64] global_step=64, grad_norm=0.499999, loss=6.90711 +I0912 02:38:39.090292 140411597268160 submission.py:307] 64) loss = 6.907, grad_norm = 0.500 +I0912 02:38:39.350845 140365738682112 logging_writer.py:48] [65] global_step=65, grad_norm=0.499999, loss=6.92607 +I0912 02:38:39.354483 140411597268160 submission.py:307] 65) loss = 6.926, grad_norm = 0.500 +I0912 02:38:39.616002 140365747074816 logging_writer.py:48] [66] global_step=66, grad_norm=0.499999, loss=6.91703 +I0912 02:38:39.619063 140411597268160 submission.py:307] 66) loss = 6.917, grad_norm = 0.500 +I0912 02:38:39.887074 140365738682112 logging_writer.py:48] [67] global_step=67, grad_norm=0.499999, loss=6.92048 +I0912 02:38:39.890105 140411597268160 submission.py:307] 67) loss = 6.920, grad_norm = 0.500 +I0912 02:38:40.150065 140365747074816 logging_writer.py:48] [68] global_step=68, grad_norm=0.499999, loss=6.91559 +I0912 02:38:40.153307 140411597268160 submission.py:307] 68) loss = 6.916, grad_norm = 0.500 +I0912 02:38:40.413673 140365738682112 logging_writer.py:48] [69] global_step=69, grad_norm=0.499999, loss=6.90996 +I0912 02:38:40.416627 140411597268160 submission.py:307] 69) loss = 6.910, grad_norm = 0.500 +I0912 02:38:40.676397 140365747074816 logging_writer.py:48] [70] global_step=70, grad_norm=0.499999, loss=6.91126 +I0912 02:38:40.679311 140411597268160 submission.py:307] 70) loss = 6.911, grad_norm = 0.500 +I0912 02:38:40.938129 140365738682112 logging_writer.py:48] [71] global_step=71, grad_norm=0.499999, loss=6.91909 +I0912 02:38:40.940920 140411597268160 submission.py:307] 71) loss = 6.919, grad_norm = 0.500 +I0912 02:38:41.199319 140365747074816 logging_writer.py:48] [72] global_step=72, grad_norm=0.499999, loss=6.90152 +I0912 02:38:41.202225 140411597268160 submission.py:307] 72) loss = 6.902, grad_norm = 0.500 +I0912 02:38:41.461254 140365738682112 logging_writer.py:48] [73] global_step=73, grad_norm=0.499999, loss=6.91619 +I0912 02:38:41.464277 140411597268160 submission.py:307] 73) loss = 6.916, grad_norm = 0.500 +I0912 02:38:41.724230 140365747074816 logging_writer.py:48] [74] global_step=74, grad_norm=0.499999, loss=6.90636 +I0912 02:38:41.727515 140411597268160 submission.py:307] 74) loss = 6.906, grad_norm = 0.500 +I0912 02:38:41.987362 140365738682112 logging_writer.py:48] [75] global_step=75, grad_norm=0.499999, loss=6.91094 +I0912 02:38:41.990484 140411597268160 submission.py:307] 75) loss = 6.911, grad_norm = 0.500 +I0912 02:38:42.250418 140365747074816 logging_writer.py:48] [76] global_step=76, grad_norm=0.499999, loss=6.90916 +I0912 02:38:42.253541 140411597268160 submission.py:307] 76) loss = 6.909, grad_norm = 0.500 +I0912 02:38:42.513445 140365738682112 logging_writer.py:48] [77] global_step=77, grad_norm=0.499999, loss=6.91733 +I0912 02:38:42.516496 140411597268160 submission.py:307] 77) loss = 6.917, grad_norm = 0.500 +I0912 02:38:42.776799 140365747074816 logging_writer.py:48] [78] global_step=78, grad_norm=0.499999, loss=6.90287 +I0912 02:38:42.779844 140411597268160 submission.py:307] 78) loss = 6.903, grad_norm = 0.500 +I0912 02:38:43.040030 140365738682112 logging_writer.py:48] [79] global_step=79, grad_norm=0.499999, loss=6.9043 +I0912 02:38:43.043076 140411597268160 submission.py:307] 79) loss = 6.904, grad_norm = 0.500 +I0912 02:38:43.303137 140365747074816 logging_writer.py:48] [80] global_step=80, grad_norm=0.499999, loss=6.90734 +I0912 02:38:43.306214 140411597268160 submission.py:307] 80) loss = 6.907, grad_norm = 0.500 +I0912 02:38:43.566268 140365738682112 logging_writer.py:48] [81] global_step=81, grad_norm=0.499999, loss=6.91185 +I0912 02:38:43.569247 140411597268160 submission.py:307] 81) loss = 6.912, grad_norm = 0.500 +I0912 02:38:43.828976 140365747074816 logging_writer.py:48] [82] global_step=82, grad_norm=0.499999, loss=6.91119 +I0912 02:38:43.831926 140411597268160 submission.py:307] 82) loss = 6.911, grad_norm = 0.500 +I0912 02:38:44.092671 140365738682112 logging_writer.py:48] [83] global_step=83, grad_norm=0.499999, loss=6.90543 +I0912 02:38:44.095762 140411597268160 submission.py:307] 83) loss = 6.905, grad_norm = 0.500 +I0912 02:38:44.357494 140365747074816 logging_writer.py:48] [84] global_step=84, grad_norm=0.499999, loss=6.9087 +I0912 02:38:44.360662 140411597268160 submission.py:307] 84) loss = 6.909, grad_norm = 0.500 +I0912 02:38:44.912037 140365738682112 logging_writer.py:48] [85] global_step=85, grad_norm=0.499999, loss=6.91426 +I0912 02:38:44.915230 140411597268160 submission.py:307] 85) loss = 6.914, grad_norm = 0.500 +I0912 02:38:46.736515 140365747074816 logging_writer.py:48] [86] global_step=86, grad_norm=0.499999, loss=6.9002 +I0912 02:38:46.739618 140411597268160 submission.py:307] 86) loss = 6.900, grad_norm = 0.500 +I0912 02:38:47.160993 140365738682112 logging_writer.py:48] [87] global_step=87, grad_norm=0.499999, loss=6.90078 +I0912 02:38:47.163943 140411597268160 submission.py:307] 87) loss = 6.901, grad_norm = 0.500 +I0912 02:38:48.044039 140365747074816 logging_writer.py:48] [88] global_step=88, grad_norm=0.499999, loss=6.90052 +I0912 02:38:48.047094 140411597268160 submission.py:307] 88) loss = 6.901, grad_norm = 0.500 +I0912 02:38:49.323218 140365738682112 logging_writer.py:48] [89] global_step=89, grad_norm=0.499999, loss=6.89638 +I0912 02:38:49.326290 140411597268160 submission.py:307] 89) loss = 6.896, grad_norm = 0.500 +I0912 02:38:49.633637 140365747074816 logging_writer.py:48] [90] global_step=90, grad_norm=0.499999, loss=6.9082 +I0912 02:38:49.636749 140411597268160 submission.py:307] 90) loss = 6.908, grad_norm = 0.500 +I0912 02:38:49.944223 140365738682112 logging_writer.py:48] [91] global_step=91, grad_norm=0.499999, loss=6.89871 +I0912 02:38:49.947265 140411597268160 submission.py:307] 91) loss = 6.899, grad_norm = 0.500 +I0912 02:38:50.821380 140365747074816 logging_writer.py:48] [92] global_step=92, grad_norm=0.499999, loss=6.89956 +I0912 02:38:50.824684 140411597268160 submission.py:307] 92) loss = 6.900, grad_norm = 0.500 +I0912 02:38:51.781520 140365738682112 logging_writer.py:48] [93] global_step=93, grad_norm=0.499999, loss=6.90244 +I0912 02:38:51.784580 140411597268160 submission.py:307] 93) loss = 6.902, grad_norm = 0.500 +I0912 02:38:52.044495 140365747074816 logging_writer.py:48] [94] global_step=94, grad_norm=0.499999, loss=6.90585 +I0912 02:38:52.047451 140411597268160 submission.py:307] 94) loss = 6.906, grad_norm = 0.500 +I0912 02:38:52.308879 140365738682112 logging_writer.py:48] [95] global_step=95, grad_norm=0.499999, loss=6.89077 +I0912 02:38:52.311914 140411597268160 submission.py:307] 95) loss = 6.891, grad_norm = 0.500 +I0912 02:38:52.572834 140365747074816 logging_writer.py:48] [96] global_step=96, grad_norm=0.499999, loss=6.89881 +I0912 02:38:52.575872 140411597268160 submission.py:307] 96) loss = 6.899, grad_norm = 0.500 +I0912 02:38:53.556813 140365738682112 logging_writer.py:48] [97] global_step=97, grad_norm=0.499999, loss=6.89056 +I0912 02:38:53.559810 140411597268160 submission.py:307] 97) loss = 6.891, grad_norm = 0.500 +I0912 02:38:53.820208 140365747074816 logging_writer.py:48] [98] global_step=98, grad_norm=0.499999, loss=6.89676 +I0912 02:38:53.823245 140411597268160 submission.py:307] 98) loss = 6.897, grad_norm = 0.500 +I0912 02:38:54.083617 140365738682112 logging_writer.py:48] [99] global_step=99, grad_norm=0.499999, loss=6.88992 +I0912 02:38:54.086741 140411597268160 submission.py:307] 99) loss = 6.890, grad_norm = 0.500 +I0912 02:38:54.347286 140365747074816 logging_writer.py:48] [100] global_step=100, grad_norm=0.499999, loss=6.90119 +I0912 02:38:54.350398 140411597268160 submission.py:307] 100) loss = 6.901, grad_norm = 0.500 +I0912 02:47:46.123917 140365738682112 logging_writer.py:48] [500] global_step=500, grad_norm=0.499999, loss=6.43374 +I0912 02:47:46.127413 140411597268160 submission.py:307] 500) loss = 6.434, grad_norm = 0.500 +I0912 02:59:00.241076 140365747074816 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.5, loss=5.89494 +I0912 02:59:00.297229 140411597268160 submission.py:307] 1000) loss = 5.895, grad_norm = 0.500 +I0912 03:06:42.219419 140365738682112 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.5, loss=5.48464 +I0912 03:06:42.223515 140411597268160 submission.py:307] 1500) loss = 5.485, grad_norm = 0.500 +I0912 03:11:38.220852 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 03:12:16.007895 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 03:13:44.075619 140411597268160 spec.py:363] Evaluating on the test split. +I0912 03:13:44.951102 140411597268160 submission_runner.py:516] Time since start: 2448.42s, Step: 1911, {'train/accuracy': 0.07583306760204081, 'train/loss': 5.683907645089286, 'validation/accuracy': 0.08448, 'validation/loss': 5.369495, 'validation/num_examples': 50000, 'test/accuracy': 0.0581, 'test/loss': 5.43460625, 'test/num_examples': 10000, 'score': 2115.945389509201, 'total_duration': 2448.4243199825287, 'accumulated_submission_time': 2115.945389509201, 'accumulated_eval_time': 329.35181760787964, 'accumulated_logging_time': 0.029718637466430664} +I0912 03:13:44.997947 140365839365888 logging_writer.py:48] [1911] accumulated_eval_time=329.352, accumulated_logging_time=0.0297186, accumulated_submission_time=2115.95, global_step=1911, preemption_count=0, score=2115.95, test/accuracy=0.0581, test/loss=5.43461, test/num_examples=10000, total_duration=2448.42, train/accuracy=0.0758331, train/loss=5.68391, validation/accuracy=0.08448, validation/loss=5.36949, validation/num_examples=50000 +I0912 03:14:10.635920 140365847758592 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.5, loss=5.12814 +I0912 03:14:10.639244 140411597268160 submission.py:307] 2000) loss = 5.128, grad_norm = 0.500 +I0912 03:23:48.854026 140365839365888 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.5, loss=4.87233 +I0912 03:23:48.857718 140411597268160 submission.py:307] 2500) loss = 4.872, grad_norm = 0.500 +I0912 03:28:00.849395 140365847758592 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.5, loss=4.49501 +I0912 03:28:00.853180 140411597268160 submission.py:307] 3000) loss = 4.495, grad_norm = 0.500 +I0912 03:34:31.384675 140365839365888 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.5, loss=4.1441 +I0912 03:34:31.388359 140411597268160 submission.py:307] 3500) loss = 4.144, grad_norm = 0.500 +I0912 03:40:20.930792 140365847758592 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.5, loss=4.10915 +I0912 03:40:20.935146 140411597268160 submission.py:307] 4000) loss = 4.109, grad_norm = 0.500 +I0912 03:44:43.052899 140365839365888 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.5, loss=3.50798 +I0912 03:44:43.056727 140411597268160 submission.py:307] 4500) loss = 3.508, grad_norm = 0.500 +I0912 03:47:06.185666 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 03:47:44.843172 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 03:49:02.638890 140411597268160 spec.py:363] Evaluating on the test split. +I0912 03:49:03.499738 140411597268160 submission_runner.py:516] Time since start: 4566.97s, Step: 4679, {'train/accuracy': 0.25296954719387754, 'train/loss': 4.5554351806640625, 'validation/accuracy': 0.2422, 'validation/loss': 4.4598834375, 'validation/num_examples': 50000, 'test/accuracy': 0.1978, 'test/loss': 4.166992578125, 'test/num_examples': 10000, 'score': 4113.932512521744, 'total_duration': 4566.972930669785, 'accumulated_submission_time': 4113.932512521744, 'accumulated_eval_time': 446.66592741012573, 'accumulated_logging_time': 0.08526349067687988} +I0912 03:49:03.527028 140390032176896 logging_writer.py:48] [4679] accumulated_eval_time=446.666, accumulated_logging_time=0.0852635, accumulated_submission_time=4113.93, global_step=4679, preemption_count=0, score=4113.93, test/accuracy=0.1978, test/loss=4.16699, test/num_examples=10000, total_duration=4566.97, train/accuracy=0.25297, train/loss=4.55544, validation/accuracy=0.2422, validation/loss=4.45988, validation/num_examples=50000 +I0912 03:53:18.030342 140390057355008 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.5, loss=3.5122 +I0912 03:53:18.033706 140411597268160 submission.py:307] 5000) loss = 3.512, grad_norm = 0.500 +I0912 03:57:03.876074 140390032176896 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.5, loss=3.25643 +I0912 03:57:03.879774 140411597268160 submission.py:307] 5500) loss = 3.256, grad_norm = 0.500 +I0912 04:03:15.567810 140390057355008 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.5, loss=3.06537 +I0912 04:03:15.571560 140411597268160 submission.py:307] 6000) loss = 3.065, grad_norm = 0.500 +I0912 04:08:49.576063 140390032176896 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.5, loss=2.8919 +I0912 04:08:49.598608 140411597268160 submission.py:307] 6500) loss = 2.892, grad_norm = 0.500 +I0912 04:12:58.402226 140390057355008 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.5, loss=2.77565 +I0912 04:12:58.405893 140411597268160 submission.py:307] 7000) loss = 2.776, grad_norm = 0.500 +I0912 04:20:22.834767 140390032176896 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.5, loss=2.69047 +I0912 04:20:22.838172 140411597268160 submission.py:307] 7500) loss = 2.690, grad_norm = 0.500 +I0912 04:22:20.737914 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 04:22:57.099447 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 04:24:04.476374 140411597268160 spec.py:363] Evaluating on the test split. +I0912 04:24:05.337543 140411597268160 submission_runner.py:516] Time since start: 6668.81s, Step: 7813, {'train/accuracy': 0.4213966836734694, 'train/loss': 4.301740373883929, 'validation/accuracy': 0.41084, 'validation/loss': 3.6116853125, 'validation/num_examples': 50000, 'test/accuracy': 0.3349, 'test/loss': 3.22026875, 'test/num_examples': 10000, 'score': 6106.99627327919, 'total_duration': 6668.810752868652, 'accumulated_submission_time': 6106.99627327919, 'accumulated_eval_time': 551.2657015323639, 'accumulated_logging_time': 0.12171149253845215} +I0912 04:24:05.375317 140389956642560 logging_writer.py:48] [7813] accumulated_eval_time=551.266, accumulated_logging_time=0.121711, accumulated_submission_time=6107, global_step=7813, preemption_count=0, score=6107, test/accuracy=0.3349, test/loss=3.22027, test/num_examples=10000, total_duration=6668.81, train/accuracy=0.421397, train/loss=4.30174, validation/accuracy=0.41084, validation/loss=3.61169, validation/num_examples=50000 +I0912 04:25:31.665254 140389998606080 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.5, loss=2.61492 +I0912 04:25:31.669346 140411597268160 submission.py:307] 8000) loss = 2.615, grad_norm = 0.500 +I0912 04:32:18.463148 140389956642560 logging_writer.py:48] [8500] global_step=8500, grad_norm=0.5, loss=2.42746 +I0912 04:32:18.466725 140411597268160 submission.py:307] 8500) loss = 2.427, grad_norm = 0.500 +I0912 04:38:03.624253 140389998606080 logging_writer.py:48] [9000] global_step=9000, grad_norm=0.5, loss=2.34893 +I0912 04:38:03.628562 140411597268160 submission.py:307] 9000) loss = 2.349, grad_norm = 0.500 +I0912 04:42:06.731182 140389956642560 logging_writer.py:48] [9500] global_step=9500, grad_norm=0.5, loss=2.29932 +I0912 04:42:06.734905 140411597268160 submission.py:307] 9500) loss = 2.299, grad_norm = 0.500 +I0912 04:49:44.175283 140389998606080 logging_writer.py:48] [10000] global_step=10000, grad_norm=0.5, loss=2.16737 +I0912 04:49:44.178716 140411597268160 submission.py:307] 10000) loss = 2.167, grad_norm = 0.500 +I0912 04:53:03.599763 140389956642560 logging_writer.py:48] [10500] global_step=10500, grad_norm=0.5, loss=2.20973 +I0912 04:53:03.603459 140411597268160 submission.py:307] 10500) loss = 2.210, grad_norm = 0.500 +I0912 04:57:22.849833 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 04:58:07.795703 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 04:59:31.852421 140411597268160 spec.py:363] Evaluating on the test split. +I0912 04:59:32.712664 140411597268160 submission_runner.py:516] Time since start: 8796.19s, Step: 10908, {'train/accuracy': 0.5089684311224489, 'train/loss': 3.690275542590083, 'validation/accuracy': 0.48904, 'validation/loss': 3.393254375, 'validation/num_examples': 50000, 'test/accuracy': 0.4208, 'test/loss': 2.76146171875, 'test/num_examples': 10000, 'score': 8100.409832239151, 'total_duration': 8796.185854911804, 'accumulated_submission_time': 8100.409832239151, 'accumulated_eval_time': 681.1285533905029, 'accumulated_logging_time': 0.16855692863464355} +I0912 04:59:32.874666 140389998606080 logging_writer.py:48] [10908] accumulated_eval_time=681.129, accumulated_logging_time=0.168557, accumulated_submission_time=8100.41, global_step=10908, preemption_count=0, score=8100.41, test/accuracy=0.4208, test/loss=2.76146, test/num_examples=10000, total_duration=8796.19, train/accuracy=0.508968, train/loss=3.69028, validation/accuracy=0.48904, validation/loss=3.39325, validation/num_examples=50000 +I0912 04:59:57.993101 140390090925824 logging_writer.py:48] [11000] global_step=11000, grad_norm=0.5, loss=2.08185 +I0912 04:59:57.996586 140411597268160 submission.py:307] 11000) loss = 2.082, grad_norm = 0.500 +I0912 05:05:54.775126 140389998606080 logging_writer.py:48] [11500] global_step=11500, grad_norm=0.5, loss=1.99497 +I0912 05:05:54.779197 140411597268160 submission.py:307] 11500) loss = 1.995, grad_norm = 0.500 +I0912 05:10:41.637249 140390090925824 logging_writer.py:48] [12000] global_step=12000, grad_norm=0.5, loss=1.93865 +I0912 05:10:41.880523 140411597268160 submission.py:307] 12000) loss = 1.939, grad_norm = 0.500 +I0912 05:18:06.822578 140389998606080 logging_writer.py:48] [12500] global_step=12500, grad_norm=0.5, loss=2.01359 +I0912 05:18:06.826203 140411597268160 submission.py:307] 12500) loss = 2.014, grad_norm = 0.500 +I0912 05:21:28.235772 140390090925824 logging_writer.py:48] [13000] global_step=13000, grad_norm=0.5, loss=1.92042 +I0912 05:21:28.239844 140411597268160 submission.py:307] 13000) loss = 1.920, grad_norm = 0.500 +I0912 05:27:37.189276 140389998606080 logging_writer.py:48] [13500] global_step=13500, grad_norm=0.5, loss=1.87883 +I0912 05:27:37.193219 140411597268160 submission.py:307] 13500) loss = 1.879, grad_norm = 0.500 +I0912 05:32:49.749197 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 05:33:26.754551 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 05:34:30.106068 140411597268160 spec.py:363] Evaluating on the test split. +I0912 05:34:30.966756 140411597268160 submission_runner.py:516] Time since start: 10894.44s, Step: 13975, {'train/accuracy': 0.5401785714285714, 'train/loss': 3.8735943229830996, 'validation/accuracy': 0.49856, 'validation/loss': 3.5946475, 'validation/num_examples': 50000, 'test/accuracy': 0.4468, 'test/loss': 2.5407837890625, 'test/num_examples': 10000, 'score': 10093.10680270195, 'total_duration': 10894.439922094345, 'accumulated_submission_time': 10093.10680270195, 'accumulated_eval_time': 782.3462271690369, 'accumulated_logging_time': 0.3399162292480469} +I0912 05:34:31.147236 140390032176896 logging_writer.py:48] [13975] accumulated_eval_time=782.346, accumulated_logging_time=0.339916, accumulated_submission_time=10093.1, global_step=13975, preemption_count=0, score=10093.1, test/accuracy=0.4468, test/loss=2.54078, test/num_examples=10000, total_duration=10894.4, train/accuracy=0.540179, train/loss=3.87359, validation/accuracy=0.49856, validation/loss=3.59465, validation/num_examples=50000 +I0912 05:34:38.444366 140387201582848 logging_writer.py:48] [14000] global_step=14000, grad_norm=0.5, loss=1.88969 +I0912 05:34:38.447612 140411597268160 submission.py:307] 14000) loss = 1.890, grad_norm = 0.500 +I0912 05:39:38.896566 140390032176896 logging_writer.py:48] [14500] global_step=14500, grad_norm=0.5, loss=1.85006 +I0912 05:39:38.900315 140411597268160 submission.py:307] 14500) loss = 1.850, grad_norm = 0.500 +I0912 05:47:37.360103 140387201582848 logging_writer.py:48] [15000] global_step=15000, grad_norm=0.5, loss=1.83286 +I0912 05:47:37.363782 140411597268160 submission.py:307] 15000) loss = 1.833, grad_norm = 0.500 +I0912 05:50:55.466366 140390032176896 logging_writer.py:48] [15500] global_step=15500, grad_norm=0.5, loss=1.56411 +I0912 05:50:55.479660 140411597268160 submission.py:307] 15500) loss = 1.564, grad_norm = 0.500 +I0912 05:57:00.021948 140387201582848 logging_writer.py:48] [16000] global_step=16000, grad_norm=0.5, loss=1.71383 +I0912 05:57:00.025738 140411597268160 submission.py:307] 16000) loss = 1.714, grad_norm = 0.500 +I0912 06:02:36.445130 140390032176896 logging_writer.py:48] [16500] global_step=16500, grad_norm=0.5, loss=1.73643 +I0912 06:02:36.449640 140411597268160 submission.py:307] 16500) loss = 1.736, grad_norm = 0.500 +I0912 06:06:42.274240 140387201582848 logging_writer.py:48] [17000] global_step=17000, grad_norm=0.5, loss=1.64329 +I0912 06:06:42.278045 140411597268160 submission.py:307] 17000) loss = 1.643, grad_norm = 0.500 +I0912 06:07:48.185406 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 06:08:37.514891 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 06:10:01.031489 140411597268160 spec.py:363] Evaluating on the test split. +I0912 06:10:01.899573 140411597268160 submission_runner.py:516] Time since start: 13025.37s, Step: 17094, {'train/accuracy': 0.5809351084183674, 'train/loss': 3.8067963269292093, 'validation/accuracy': 0.55234, 'validation/loss': 3.24312875, 'validation/num_examples': 50000, 'test/accuracy': 0.4778, 'test/loss': 2.466045703125, 'test/num_examples': 10000, 'score': 12085.535720586777, 'total_duration': 13025.37265920639, 'accumulated_submission_time': 12085.535720586777, 'accumulated_eval_time': 916.0604431629181, 'accumulated_logging_time': 0.5296454429626465} +I0912 06:10:02.068071 140389948249856 logging_writer.py:48] [17094] accumulated_eval_time=916.06, accumulated_logging_time=0.529645, accumulated_submission_time=12085.5, global_step=17094, preemption_count=0, score=12085.5, test/accuracy=0.4778, test/loss=2.46605, test/num_examples=10000, total_duration=13025.4, train/accuracy=0.580935, train/loss=3.8068, validation/accuracy=0.55234, validation/loss=3.24313, validation/num_examples=50000 +I0912 06:15:45.113210 140389956642560 logging_writer.py:48] [17500] global_step=17500, grad_norm=0.5, loss=1.64958 +I0912 06:15:45.117088 140411597268160 submission.py:307] 17500) loss = 1.650, grad_norm = 0.500 +I0912 06:19:38.560119 140389948249856 logging_writer.py:48] [18000] global_step=18000, grad_norm=0.5, loss=1.65079 +I0912 06:19:38.563951 140411597268160 submission.py:307] 18000) loss = 1.651, grad_norm = 0.500 +I0912 06:25:44.914324 140389956642560 logging_writer.py:48] [18500] global_step=18500, grad_norm=0.5, loss=1.64359 +I0912 06:25:44.918113 140411597268160 submission.py:307] 18500) loss = 1.644, grad_norm = 0.500 +I0912 06:31:22.554381 140389948249856 logging_writer.py:48] [19000] global_step=19000, grad_norm=0.5, loss=1.57858 +I0912 06:31:22.558492 140411597268160 submission.py:307] 19000) loss = 1.579, grad_norm = 0.500 +I0912 06:35:30.080452 140389956642560 logging_writer.py:48] [19500] global_step=19500, grad_norm=0.5, loss=1.48226 +I0912 06:35:30.084338 140411597268160 submission.py:307] 19500) loss = 1.482, grad_norm = 0.500 +I0912 06:43:06.606143 140389948249856 logging_writer.py:48] [20000] global_step=20000, grad_norm=0.5, loss=1.51576 +I0912 06:43:06.609808 140411597268160 submission.py:307] 20000) loss = 1.516, grad_norm = 0.500 +I0912 06:43:22.338344 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 06:43:56.813261 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 06:45:00.156944 140411597268160 spec.py:363] Evaluating on the test split. +I0912 06:45:01.016211 140411597268160 submission_runner.py:516] Time since start: 15124.49s, Step: 20017, {'train/accuracy': 0.5847417091836735, 'train/loss': 4.2652899294483415, 'validation/accuracy': 0.57624, 'validation/loss': 3.0346875, 'validation/num_examples': 50000, 'test/accuracy': 0.4969, 'test/loss': 2.292778125, 'test/num_examples': 10000, 'score': 14080.839374542236, 'total_duration': 15124.489433050156, 'accumulated_submission_time': 14080.839374542236, 'accumulated_eval_time': 1014.7383263111115, 'accumulated_logging_time': 0.7073013782501221} +I0912 06:45:01.346475 140390015391488 logging_writer.py:48] [20017] accumulated_eval_time=1014.74, accumulated_logging_time=0.707301, accumulated_submission_time=14080.8, global_step=20017, preemption_count=0, score=14080.8, test/accuracy=0.4969, test/loss=2.29278, test/num_examples=10000, total_duration=15124.5, train/accuracy=0.584742, train/loss=4.26529, validation/accuracy=0.57624, validation/loss=3.03469, validation/num_examples=50000 +I0912 06:48:41.889471 140389956642560 logging_writer.py:48] [20500] global_step=20500, grad_norm=0.5, loss=1.58148 +I0912 06:48:41.893105 140411597268160 submission.py:307] 20500) loss = 1.581, grad_norm = 0.500 +I0912 06:55:14.242882 140390015391488 logging_writer.py:48] [21000] global_step=21000, grad_norm=0.5, loss=1.55771 +I0912 06:55:14.246562 140411597268160 submission.py:307] 21000) loss = 1.558, grad_norm = 0.500 +I0912 07:00:59.930811 140389956642560 logging_writer.py:48] [21500] global_step=21500, grad_norm=0.5, loss=1.57106 +I0912 07:00:59.935037 140411597268160 submission.py:307] 21500) loss = 1.571, grad_norm = 0.500 +I0912 07:05:03.747782 140390015391488 logging_writer.py:48] [22000] global_step=22000, grad_norm=0.5, loss=1.57535 +I0912 07:05:03.751996 140411597268160 submission.py:307] 22000) loss = 1.575, grad_norm = 0.500 +I0912 07:12:49.273973 140389956642560 logging_writer.py:48] [22500] global_step=22500, grad_norm=0.5, loss=1.59154 +I0912 07:12:49.277454 140411597268160 submission.py:307] 22500) loss = 1.592, grad_norm = 0.500 +I0912 07:16:04.824135 140390015391488 logging_writer.py:48] [23000] global_step=23000, grad_norm=0.5, loss=1.50118 +I0912 07:16:04.830001 140411597268160 submission.py:307] 23000) loss = 1.501, grad_norm = 0.500 +I0912 07:18:23.155270 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 07:19:13.467746 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 07:20:24.017364 140411597268160 spec.py:363] Evaluating on the test split. +I0912 07:20:24.878654 140411597268160 submission_runner.py:516] Time since start: 17248.35s, Step: 23241, {'train/accuracy': 0.6491749043367347, 'train/loss': 3.4248445471938775, 'validation/accuracy': 0.57928, 'validation/loss': 3.2999325, 'validation/num_examples': 50000, 'test/accuracy': 0.5222, 'test/loss': 2.24196796875, 'test/num_examples': 10000, 'score': 16076.302437067032, 'total_duration': 17248.351852178574, 'accumulated_submission_time': 16076.302437067032, 'accumulated_eval_time': 1136.4618639945984, 'accumulated_logging_time': 1.0467360019683838} +I0912 07:20:25.158334 140390015391488 logging_writer.py:48] [23241] accumulated_eval_time=1136.46, accumulated_logging_time=1.04674, accumulated_submission_time=16076.3, global_step=23241, preemption_count=0, score=16076.3, test/accuracy=0.5222, test/loss=2.24197, test/num_examples=10000, total_duration=17248.4, train/accuracy=0.649175, train/loss=3.42484, validation/accuracy=0.57928, validation/loss=3.29993, validation/num_examples=50000 +I0912 07:23:26.238917 140389956642560 logging_writer.py:48] [23500] global_step=23500, grad_norm=0.5, loss=1.4331 +I0912 07:23:26.242650 140411597268160 submission.py:307] 23500) loss = 1.433, grad_norm = 0.500 +I0912 07:29:25.694653 140390015391488 logging_writer.py:48] [24000] global_step=24000, grad_norm=0.5, loss=1.37061 +I0912 07:29:25.698428 140411597268160 submission.py:307] 24000) loss = 1.371, grad_norm = 0.500 +I0912 07:33:56.169110 140389956642560 logging_writer.py:48] [24500] global_step=24500, grad_norm=0.5, loss=1.43221 +I0912 07:33:56.172845 140411597268160 submission.py:307] 24500) loss = 1.432, grad_norm = 0.500 +I0912 07:41:34.140637 140390015391488 logging_writer.py:48] [25000] global_step=25000, grad_norm=0.5, loss=1.48358 +I0912 07:41:34.144263 140411597268160 submission.py:307] 25000) loss = 1.484, grad_norm = 0.500 +I0912 07:44:49.670446 140389956642560 logging_writer.py:48] [25500] global_step=25500, grad_norm=0.5, loss=1.3948 +I0912 07:44:49.674674 140411597268160 submission.py:307] 25500) loss = 1.395, grad_norm = 0.500 +I0912 07:50:40.910932 140390015391488 logging_writer.py:48] [26000] global_step=26000, grad_norm=0.5, loss=1.37053 +I0912 07:50:40.914976 140411597268160 submission.py:307] 26000) loss = 1.371, grad_norm = 0.500 +I0912 07:53:43.412984 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 07:54:16.253231 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 07:55:37.937105 140411597268160 spec.py:363] Evaluating on the test split. +I0912 07:55:38.797984 140411597268160 submission_runner.py:516] Time since start: 19362.27s, Step: 26171, {'train/accuracy': 0.6524035395408163, 'train/loss': 3.653707387496014, 'validation/accuracy': 0.58686, 'validation/loss': 3.1516565625, 'validation/num_examples': 50000, 'test/accuracy': 0.5141, 'test/loss': 2.249761328125, 'test/num_examples': 10000, 'score': 18069.759664535522, 'total_duration': 19362.27117872238, 'accumulated_submission_time': 18069.759664535522, 'accumulated_eval_time': 1251.8468353748322, 'accumulated_logging_time': 1.3355841636657715} +I0912 07:55:39.078921 140390032176896 logging_writer.py:48] [26171] accumulated_eval_time=1251.85, accumulated_logging_time=1.33558, accumulated_submission_time=18069.8, global_step=26171, preemption_count=0, score=18069.8, test/accuracy=0.5141, test/loss=2.24976, test/num_examples=10000, total_duration=19362.3, train/accuracy=0.652404, train/loss=3.65371, validation/accuracy=0.58686, validation/loss=3.15166, validation/num_examples=50000 +I0912 07:58:06.174318 140390074140416 logging_writer.py:48] [26500] global_step=26500, grad_norm=0.5, loss=1.42252 +I0912 07:58:06.178192 140411597268160 submission.py:307] 26500) loss = 1.423, grad_norm = 0.500 +I0912 08:02:56.178029 140390032176896 logging_writer.py:48] [27000] global_step=27000, grad_norm=0.5, loss=1.34756 +I0912 08:02:56.181740 140411597268160 submission.py:307] 27000) loss = 1.348, grad_norm = 0.500 +I0912 08:10:45.957686 140390074140416 logging_writer.py:48] [27500] global_step=27500, grad_norm=0.5, loss=1.38758 +I0912 08:10:45.961304 140411597268160 submission.py:307] 27500) loss = 1.388, grad_norm = 0.500 +I0912 08:13:59.211448 140390032176896 logging_writer.py:48] [28000] global_step=28000, grad_norm=0.5, loss=1.26935 +I0912 08:13:59.215339 140411597268160 submission.py:307] 28000) loss = 1.269, grad_norm = 0.500 +I0912 08:19:58.922701 140390074140416 logging_writer.py:48] [28500] global_step=28500, grad_norm=0.5, loss=1.34166 +I0912 08:19:58.926676 140411597268160 submission.py:307] 28500) loss = 1.342, grad_norm = 0.500 +I0912 08:25:45.745038 140390032176896 logging_writer.py:48] [29000] global_step=29000, grad_norm=0.5, loss=1.3954 +I0912 08:25:45.749183 140411597268160 submission.py:307] 29000) loss = 1.395, grad_norm = 0.500 +I0912 08:28:58.422637 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 08:29:49.711627 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 08:30:57.344729 140411597268160 spec.py:363] Evaluating on the test split. +I0912 08:30:58.207037 140411597268160 submission_runner.py:516] Time since start: 21481.68s, Step: 29427, {'train/accuracy': 0.6569674744897959, 'train/loss': 3.933512784996811, 'validation/accuracy': 0.5849, 'validation/loss': 3.3899871875, 'validation/num_examples': 50000, 'test/accuracy': 0.5272, 'test/loss': 2.18953828125, 'test/num_examples': 10000, 'score': 20062.942791700363, 'total_duration': 21481.680223941803, 'accumulated_submission_time': 20062.942791700363, 'accumulated_eval_time': 1371.63134431839, 'accumulated_logging_time': 1.625967264175415} +I0912 08:30:58.556024 140390074140416 logging_writer.py:48] [29427] accumulated_eval_time=1371.63, accumulated_logging_time=1.62597, accumulated_submission_time=20062.9, global_step=29427, preemption_count=0, score=20062.9, test/accuracy=0.5272, test/loss=2.18954, test/num_examples=10000, total_duration=21481.7, train/accuracy=0.656967, train/loss=3.93351, validation/accuracy=0.5849, validation/loss=3.38999, validation/num_examples=50000 +I0912 08:31:17.662498 140390048962304 logging_writer.py:48] [29500] global_step=29500, grad_norm=0.5, loss=1.29524 +I0912 08:31:17.665755 140411597268160 submission.py:307] 29500) loss = 1.295, grad_norm = 0.500 +I0912 08:39:13.066409 140390074140416 logging_writer.py:48] [30000] global_step=30000, grad_norm=0.5, loss=1.22878 +I0912 08:39:13.070300 140411597268160 submission.py:307] 30000) loss = 1.229, grad_norm = 0.500 +I0912 08:42:59.630696 140390048962304 logging_writer.py:48] [30500] global_step=30500, grad_norm=0.5, loss=1.28857 +I0912 08:42:59.649686 140411597268160 submission.py:307] 30500) loss = 1.289, grad_norm = 0.500 +I0912 08:49:00.883738 140390074140416 logging_writer.py:48] [31000] global_step=31000, grad_norm=0.5, loss=1.20288 +I0912 08:49:00.887800 140411597268160 submission.py:307] 31000) loss = 1.203, grad_norm = 0.500 +I0912 08:54:41.180439 140390048962304 logging_writer.py:48] [31500] global_step=31500, grad_norm=0.5, loss=1.24869 +I0912 08:54:41.184336 140411597268160 submission.py:307] 31500) loss = 1.249, grad_norm = 0.500 +I0912 08:58:42.249289 140390074140416 logging_writer.py:48] [32000] global_step=32000, grad_norm=0.5, loss=1.24254 +I0912 08:58:42.261965 140411597268160 submission.py:307] 32000) loss = 1.243, grad_norm = 0.500 +I0912 09:04:22.461272 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 09:04:57.132308 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 09:06:19.475575 140411597268160 spec.py:363] Evaluating on the test split. +I0912 09:06:20.339448 140411597268160 submission_runner.py:516] Time since start: 23603.81s, Step: 32376, {'train/accuracy': 0.6981425382653061, 'train/loss': 3.257666451590402, 'validation/accuracy': 0.60376, 'validation/loss': 3.2073090625, 'validation/num_examples': 50000, 'test/accuracy': 0.5364, 'test/loss': 2.1774421875, 'test/num_examples': 10000, 'score': 22062.18269753456, 'total_duration': 23603.81263756752, 'accumulated_submission_time': 22062.18269753456, 'accumulated_eval_time': 1489.5095262527466, 'accumulated_logging_time': 1.9843802452087402} +I0912 09:06:20.628329 140390006998784 logging_writer.py:48] [32376] accumulated_eval_time=1489.51, accumulated_logging_time=1.98438, accumulated_submission_time=22062.2, global_step=32376, preemption_count=0, score=22062.2, test/accuracy=0.5364, test/loss=2.17744, test/num_examples=10000, total_duration=23603.8, train/accuracy=0.698143, train/loss=3.25767, validation/accuracy=0.60376, validation/loss=3.20731, validation/num_examples=50000 +I0912 09:07:25.083255 140390032176896 logging_writer.py:48] [32500] global_step=32500, grad_norm=0.5, loss=1.39656 +I0912 09:07:25.086865 140411597268160 submission.py:307] 32500) loss = 1.397, grad_norm = 0.500 +I0912 09:11:24.388059 140390006998784 logging_writer.py:48] [33000] global_step=33000, grad_norm=0.5, loss=1.31089 +I0912 09:11:24.392092 140411597268160 submission.py:307] 33000) loss = 1.311, grad_norm = 0.500 +I0912 09:17:43.949941 140390032176896 logging_writer.py:48] [33500] global_step=33500, grad_norm=0.5, loss=1.30154 +I0912 09:17:43.953905 140411597268160 submission.py:307] 33500) loss = 1.302, grad_norm = 0.500 +I0912 09:23:30.555980 140390006998784 logging_writer.py:48] [34000] global_step=34000, grad_norm=0.5, loss=1.19926 +I0912 09:23:30.560881 140411597268160 submission.py:307] 34000) loss = 1.199, grad_norm = 0.500 +I0912 09:27:29.165418 140390032176896 logging_writer.py:48] [34500] global_step=34500, grad_norm=0.5, loss=1.18876 +I0912 09:27:29.169198 140411597268160 submission.py:307] 34500) loss = 1.189, grad_norm = 0.500 +I0912 09:35:17.686077 140390006998784 logging_writer.py:48] [35000] global_step=35000, grad_norm=0.5, loss=1.1084 +I0912 09:35:17.689872 140411597268160 submission.py:307] 35000) loss = 1.108, grad_norm = 0.500 +I0912 09:38:33.608143 140390032176896 logging_writer.py:48] [35500] global_step=35500, grad_norm=0.5, loss=1.13249 +I0912 09:38:33.612173 140411597268160 submission.py:307] 35500) loss = 1.132, grad_norm = 0.500 +I0912 09:39:41.115972 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 09:40:29.560142 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 09:41:34.767449 140411597268160 spec.py:363] Evaluating on the test split. +I0912 09:41:35.628476 140411597268160 submission_runner.py:516] Time since start: 25719.10s, Step: 35628, {'train/accuracy': 0.7033641581632653, 'train/loss': 3.4684765874123085, 'validation/accuracy': 0.59992, 'validation/loss': 3.4122078125, 'validation/num_examples': 50000, 'test/accuracy': 0.5405, 'test/loss': 2.2027775390625, 'test/num_examples': 10000, 'score': 24057.024208545685, 'total_duration': 25719.101647377014, 'accumulated_submission_time': 24057.024208545685, 'accumulated_eval_time': 1604.0221438407898, 'accumulated_logging_time': 2.28261399269104} +I0912 09:41:35.896652 140390074140416 logging_writer.py:48] [35628] accumulated_eval_time=1604.02, accumulated_logging_time=2.28261, accumulated_submission_time=24057, global_step=35628, preemption_count=0, score=24057, test/accuracy=0.5405, test/loss=2.20278, test/num_examples=10000, total_duration=25719.1, train/accuracy=0.703364, train/loss=3.46848, validation/accuracy=0.59992, validation/loss=3.41221, validation/num_examples=50000 +I0912 09:46:24.672150 140389981820672 logging_writer.py:48] [36000] global_step=36000, grad_norm=0.5, loss=1.21751 +I0912 09:46:24.676049 140411597268160 submission.py:307] 36000) loss = 1.218, grad_norm = 0.500 +I0912 09:52:32.382977 140390074140416 logging_writer.py:48] [36500] global_step=36500, grad_norm=0.5, loss=1.21008 +I0912 09:52:32.386729 140411597268160 submission.py:307] 36500) loss = 1.210, grad_norm = 0.500 +I0912 09:56:40.369260 140389981820672 logging_writer.py:48] [37000] global_step=37000, grad_norm=0.5, loss=1.26711 +I0912 09:56:40.372937 140411597268160 submission.py:307] 37000) loss = 1.267, grad_norm = 0.500 +I0912 10:04:20.785290 140390074140416 logging_writer.py:48] [37500] global_step=37500, grad_norm=0.5, loss=1.23464 +I0912 10:04:20.789542 140411597268160 submission.py:307] 37500) loss = 1.235, grad_norm = 0.500 +I0912 10:07:39.496489 140389981820672 logging_writer.py:48] [38000] global_step=38000, grad_norm=0.5, loss=1.18985 +I0912 10:07:39.506016 140411597268160 submission.py:307] 38000) loss = 1.190, grad_norm = 0.500 +I0912 10:13:25.123973 140390074140416 logging_writer.py:48] [38500] global_step=38500, grad_norm=0.5, loss=1.19881 +I0912 10:13:25.127667 140411597268160 submission.py:307] 38500) loss = 1.199, grad_norm = 0.500 +I0912 10:14:59.146352 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 10:15:35.438714 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 10:16:57.590619 140411597268160 spec.py:363] Evaluating on the test split. +I0912 10:16:58.451422 140411597268160 submission_runner.py:516] Time since start: 27841.92s, Step: 38590, {'train/accuracy': 0.7145448022959183, 'train/loss': 3.2748126594387754, 'validation/accuracy': 0.58636, 'validation/loss': 3.774430625, 'validation/num_examples': 50000, 'test/accuracy': 0.5371, 'test/loss': 2.18135625, 'test/num_examples': 10000, 'score': 26055.084835529327, 'total_duration': 27841.924547433853, 'accumulated_submission_time': 26055.084835529327, 'accumulated_eval_time': 1723.3271679878235, 'accumulated_logging_time': 2.5602216720581055} +I0912 10:16:58.716041 140390074140416 logging_writer.py:48] [38590] accumulated_eval_time=1723.33, accumulated_logging_time=2.56022, accumulated_submission_time=26055.1, global_step=38590, preemption_count=0, score=26055.1, test/accuracy=0.5371, test/loss=2.18136, test/num_examples=10000, total_duration=27841.9, train/accuracy=0.714545, train/loss=3.27481, validation/accuracy=0.58636, validation/loss=3.77443, validation/num_examples=50000 +I0912 10:20:36.792065 140389981820672 logging_writer.py:48] [39000] global_step=39000, grad_norm=0.5, loss=1.1588 +I0912 10:20:36.795934 140411597268160 submission.py:307] 39000) loss = 1.159, grad_norm = 0.500 +I0912 10:25:19.440005 140390074140416 logging_writer.py:48] [39500] global_step=39500, grad_norm=0.5, loss=1.13516 +I0912 10:25:19.443716 140411597268160 submission.py:307] 39500) loss = 1.135, grad_norm = 0.500 +I0912 10:32:53.599249 140389981820672 logging_writer.py:48] [40000] global_step=40000, grad_norm=0.5, loss=1.16973 +I0912 10:32:53.603075 140411597268160 submission.py:307] 40000) loss = 1.170, grad_norm = 0.500 +I0912 10:36:11.743956 140390074140416 logging_writer.py:48] [40500] global_step=40500, grad_norm=0.5, loss=1.12799 +I0912 10:36:11.747908 140411597268160 submission.py:307] 40500) loss = 1.128, grad_norm = 0.500 +I0912 10:42:12.952219 140389981820672 logging_writer.py:48] [41000] global_step=41000, grad_norm=0.5, loss=1.09516 +I0912 10:42:12.956135 140411597268160 submission.py:307] 41000) loss = 1.095, grad_norm = 0.500 +I0912 10:47:55.406685 140390074140416 logging_writer.py:48] [41500] global_step=41500, grad_norm=0.5, loss=1.04219 +I0912 10:47:55.426812 140411597268160 submission.py:307] 41500) loss = 1.042, grad_norm = 0.500 +I0912 10:50:16.815170 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 10:51:02.569355 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 10:52:08.109826 140411597268160 spec.py:363] Evaluating on the test split. +I0912 10:52:08.969816 140411597268160 submission_runner.py:516] Time since start: 29952.44s, Step: 41831, {'train/accuracy': 0.7114158163265306, 'train/loss': 3.6007372797751915, 'validation/accuracy': 0.60598, 'validation/loss': 3.3401575, 'validation/num_examples': 50000, 'test/accuracy': 0.5336, 'test/loss': 2.324109765625, 'test/num_examples': 10000, 'score': 28047.74717950821, 'total_duration': 29952.44298005104, 'accumulated_submission_time': 28047.74717950821, 'accumulated_eval_time': 1835.4817848205566, 'accumulated_logging_time': 2.8340516090393066} +I0912 10:52:09.233293 140390057355008 logging_writer.py:48] [41831] accumulated_eval_time=1835.48, accumulated_logging_time=2.83405, accumulated_submission_time=28047.7, global_step=41831, preemption_count=0, score=28047.7, test/accuracy=0.5336, test/loss=2.32411, test/num_examples=10000, total_duration=29952.4, train/accuracy=0.711416, train/loss=3.60074, validation/accuracy=0.60598, validation/loss=3.34016, validation/num_examples=50000 +I0912 10:53:41.870361 140387592111872 logging_writer.py:48] [42000] global_step=42000, grad_norm=0.5, loss=1.16584 +I0912 10:53:41.873884 140411597268160 submission.py:307] 42000) loss = 1.166, grad_norm = 0.500 +I0912 11:01:45.830002 140390057355008 logging_writer.py:48] [42500] global_step=42500, grad_norm=0.5, loss=1.15302 +I0912 11:01:45.833853 140411597268160 submission.py:307] 42500) loss = 1.153, grad_norm = 0.500 +I0912 11:05:29.741174 140387592111872 logging_writer.py:48] [43000] global_step=43000, grad_norm=0.5, loss=1.08829 +I0912 11:05:29.745390 140411597268160 submission.py:307] 43000) loss = 1.088, grad_norm = 0.500 +I0912 11:11:18.825171 140390057355008 logging_writer.py:48] [43500] global_step=43500, grad_norm=0.499999, loss=1.01242 +I0912 11:11:18.828914 140411597268160 submission.py:307] 43500) loss = 1.012, grad_norm = 0.500 +I0912 11:17:08.680450 140387592111872 logging_writer.py:48] [44000] global_step=44000, grad_norm=0.5, loss=1.18415 +I0912 11:17:08.709759 140411597268160 submission.py:307] 44000) loss = 1.184, grad_norm = 0.500 +I0912 11:21:07.674860 140390057355008 logging_writer.py:48] [44500] global_step=44500, grad_norm=0.5, loss=1.13658 +I0912 11:21:07.678689 140411597268160 submission.py:307] 44500) loss = 1.137, grad_norm = 0.500 +I0912 11:25:30.011585 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 11:26:06.703409 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 11:27:32.962009 140411597268160 spec.py:363] Evaluating on the test split. +I0912 11:27:33.821370 140411597268160 submission_runner.py:516] Time since start: 32077.29s, Step: 44812, {'train/accuracy': 0.6891940369897959, 'train/loss': 4.2108745964206, 'validation/accuracy': 0.60408, 'validation/loss': 3.409203125, 'validation/num_examples': 50000, 'test/accuracy': 0.5384, 'test/loss': 2.2525880859375, 'test/num_examples': 10000, 'score': 30044.303684949875, 'total_duration': 32077.29454255104, 'accumulated_submission_time': 30044.303684949875, 'accumulated_eval_time': 1959.2915556430817, 'accumulated_logging_time': 3.106719970703125} +I0912 11:27:34.145036 140390023784192 logging_writer.py:48] [44812] accumulated_eval_time=1959.29, accumulated_logging_time=3.10672, accumulated_submission_time=30044.3, global_step=44812, preemption_count=0, score=30044.3, test/accuracy=0.5384, test/loss=2.25259, test/num_examples=10000, total_duration=32077.3, train/accuracy=0.689194, train/loss=4.21087, validation/accuracy=0.60408, validation/loss=3.4092, validation/num_examples=50000 +I0912 11:29:43.727505 140390090925824 logging_writer.py:48] [45000] global_step=45000, grad_norm=0.5, loss=1.10668 +I0912 11:29:43.731285 140411597268160 submission.py:307] 45000) loss = 1.107, grad_norm = 0.500 +I0912 11:33:42.819882 140390023784192 logging_writer.py:48] [45500] global_step=45500, grad_norm=0.5, loss=1.04179 +I0912 11:33:42.824192 140411597268160 submission.py:307] 45500) loss = 1.042, grad_norm = 0.500 +I0912 11:39:58.769370 140390090925824 logging_writer.py:48] [46000] global_step=46000, grad_norm=0.5, loss=1.09551 +I0912 11:39:58.772987 140411597268160 submission.py:307] 46000) loss = 1.096, grad_norm = 0.500 +I0912 11:45:37.915518 140390023784192 logging_writer.py:48] [46500] global_step=46500, grad_norm=0.5, loss=1.16539 +I0912 11:45:37.941281 140411597268160 submission.py:307] 46500) loss = 1.165, grad_norm = 0.500 +I0912 11:49:32.425969 140390090925824 logging_writer.py:48] [47000] global_step=47000, grad_norm=0.5, loss=1.16554 +I0912 11:49:32.429961 140411597268160 submission.py:307] 47000) loss = 1.166, grad_norm = 0.500 +I0912 11:57:18.975907 140390023784192 logging_writer.py:48] [47500] global_step=47500, grad_norm=0.5, loss=1.11733 +I0912 11:57:18.979775 140411597268160 submission.py:307] 47500) loss = 1.117, grad_norm = 0.500 +I0912 12:00:38.738486 140390090925824 logging_writer.py:48] [48000] global_step=48000, grad_norm=0.5, loss=1.0503 +I0912 12:00:38.749795 140411597268160 submission.py:307] 48000) loss = 1.050, grad_norm = 0.500 +I0912 12:00:52.449037 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 12:01:37.086400 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 12:02:41.366892 140411597268160 spec.py:363] Evaluating on the test split. +I0912 12:02:42.229760 140411597268160 submission_runner.py:516] Time since start: 34185.70s, Step: 48029, {'train/accuracy': 0.7480269451530612, 'train/loss': 3.264349489795918, 'validation/accuracy': 0.61424, 'validation/loss': 3.2800534375, 'validation/num_examples': 50000, 'test/accuracy': 0.5376, 'test/loss': 2.3202591796875, 'test/num_examples': 10000, 'score': 32036.41220641136, 'total_duration': 34185.70292234421, 'accumulated_submission_time': 32036.41220641136, 'accumulated_eval_time': 2069.0723991394043, 'accumulated_logging_time': 3.439774990081787} +I0912 12:02:42.476204 140390099318528 logging_writer.py:48] [48029] accumulated_eval_time=2069.07, accumulated_logging_time=3.43977, accumulated_submission_time=32036.4, global_step=48029, preemption_count=0, score=32036.4, test/accuracy=0.5376, test/loss=2.32026, test/num_examples=10000, total_duration=34185.7, train/accuracy=0.748027, train/loss=3.26435, validation/accuracy=0.61424, validation/loss=3.28005, validation/num_examples=50000 +I0912 12:08:47.799653 140390065747712 logging_writer.py:48] [48500] global_step=48500, grad_norm=0.5, loss=1.0829 +I0912 12:08:47.803405 140411597268160 submission.py:307] 48500) loss = 1.083, grad_norm = 0.500 +I0912 12:14:58.389255 140390099318528 logging_writer.py:48] [49000] global_step=49000, grad_norm=0.5, loss=0.97979 +I0912 12:14:58.398181 140411597268160 submission.py:307] 49000) loss = 0.980, grad_norm = 0.500 +I0912 12:18:53.610754 140390065747712 logging_writer.py:48] [49500] global_step=49500, grad_norm=0.5, loss=1.01272 +I0912 12:18:53.614845 140411597268160 submission.py:307] 49500) loss = 1.013, grad_norm = 0.500 +I0912 12:26:42.442766 140390099318528 logging_writer.py:48] [50000] global_step=50000, grad_norm=0.5, loss=1.03465 +I0912 12:26:42.446735 140411597268160 submission.py:307] 50000) loss = 1.035, grad_norm = 0.500 +I0912 12:30:01.584906 140390065747712 logging_writer.py:48] [50500] global_step=50500, grad_norm=0.5, loss=1.02815 +I0912 12:30:01.589099 140411597268160 submission.py:307] 50500) loss = 1.028, grad_norm = 0.500 +I0912 12:35:28.667245 140390099318528 logging_writer.py:48] [51000] global_step=51000, grad_norm=0.5, loss=1.03032 +I0912 12:35:28.671214 140411597268160 submission.py:307] 51000) loss = 1.030, grad_norm = 0.500 +I0912 12:36:05.920046 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 12:36:44.760134 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 12:38:05.309909 140411597268160 spec.py:363] Evaluating on the test split. +I0912 12:38:06.171114 140411597268160 submission_runner.py:516] Time since start: 36309.64s, Step: 51025, {'train/accuracy': 0.7228156887755102, 'train/loss': 4.107982090541294, 'validation/accuracy': 0.60628, 'validation/loss': 3.646350625, 'validation/num_examples': 50000, 'test/accuracy': 0.5283, 'test/loss': 2.405058984375, 'test/num_examples': 10000, 'score': 34034.62011766434, 'total_duration': 36309.644293785095, 'accumulated_submission_time': 34034.62011766434, 'accumulated_eval_time': 2189.3234827518463, 'accumulated_logging_time': 3.695246696472168} +I0912 12:38:06.438312 140390040569600 logging_writer.py:48] [51025] accumulated_eval_time=2189.32, accumulated_logging_time=3.69525, accumulated_submission_time=34034.6, global_step=51025, preemption_count=0, score=34034.6, test/accuracy=0.5283, test/loss=2.40506, test/num_examples=10000, total_duration=36309.6, train/accuracy=0.722816, train/loss=4.10798, validation/accuracy=0.60628, validation/loss=3.64635, validation/num_examples=50000 +I0912 12:42:53.748374 140389965035264 logging_writer.py:48] [51500] global_step=51500, grad_norm=0.5, loss=0.928383 +I0912 12:42:53.752352 140411597268160 submission.py:307] 51500) loss = 0.928, grad_norm = 0.500 +I0912 12:47:35.377418 140390040569600 logging_writer.py:48] [52000] global_step=52000, grad_norm=0.5, loss=0.985892 +I0912 12:47:35.381439 140411597268160 submission.py:307] 52000) loss = 0.986, grad_norm = 0.500 +I0912 12:55:07.519468 140389965035264 logging_writer.py:48] [52500] global_step=52500, grad_norm=0.5, loss=1.10443 +I0912 12:55:07.523492 140411597268160 submission.py:307] 52500) loss = 1.104, grad_norm = 0.500 +I0912 12:58:30.884187 140390040569600 logging_writer.py:48] [53000] global_step=53000, grad_norm=0.5, loss=0.923953 +I0912 12:58:30.888213 140411597268160 submission.py:307] 53000) loss = 0.924, grad_norm = 0.500 +I0912 13:04:17.658897 140389965035264 logging_writer.py:48] [53500] global_step=53500, grad_norm=0.5, loss=1.02128 +I0912 13:04:17.662841 140411597268160 submission.py:307] 53500) loss = 1.021, grad_norm = 0.500 +I0912 13:10:15.694733 140390040569600 logging_writer.py:48] [54000] global_step=54000, grad_norm=0.5, loss=1.03617 +I0912 13:10:15.700197 140411597268160 submission.py:307] 54000) loss = 1.036, grad_norm = 0.500 +I0912 13:11:26.522406 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 13:12:03.528717 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 13:13:07.376857 140411597268160 spec.py:363] Evaluating on the test split. +I0912 13:13:08.267889 140411597268160 submission_runner.py:516] Time since start: 38411.74s, Step: 54173, {'train/accuracy': 0.7489038584183674, 'train/loss': 3.6770060013751595, 'validation/accuracy': 0.61914, 'validation/loss': 3.423630625, 'validation/num_examples': 50000, 'test/accuracy': 0.5371, 'test/loss': 2.37350625, 'test/num_examples': 10000, 'score': 36030.59577226639, 'total_duration': 38411.74111676216, 'accumulated_submission_time': 36030.59577226639, 'accumulated_eval_time': 2291.069113969803, 'accumulated_logging_time': 3.971768617630005} +I0912 13:13:08.600744 140389973427968 logging_writer.py:48] [54173] accumulated_eval_time=2291.07, accumulated_logging_time=3.97177, accumulated_submission_time=36030.6, global_step=54173, preemption_count=0, score=36030.6, test/accuracy=0.5371, test/loss=2.37351, test/num_examples=10000, total_duration=38411.7, train/accuracy=0.748904, train/loss=3.67701, validation/accuracy=0.61914, validation/loss=3.42363, validation/num_examples=50000 +I0912 13:16:32.585056 140389990213376 logging_writer.py:48] [54500] global_step=54500, grad_norm=0.5, loss=1.02592 +I0912 13:16:32.588675 140411597268160 submission.py:307] 54500) loss = 1.026, grad_norm = 0.500 +I0912 13:24:24.783674 140389973427968 logging_writer.py:48] [55000] global_step=55000, grad_norm=0.5, loss=1.02704 +I0912 13:24:24.787261 140411597268160 submission.py:307] 55000) loss = 1.027, grad_norm = 0.500 +I0912 13:27:57.065122 140389990213376 logging_writer.py:48] [55500] global_step=55500, grad_norm=0.5, loss=0.896029 +I0912 13:27:57.069220 140411597268160 submission.py:307] 55500) loss = 0.896, grad_norm = 0.500 +I0912 13:33:43.842901 140389973427968 logging_writer.py:48] [56000] global_step=56000, grad_norm=0.5, loss=1.02966 +I0912 13:33:43.847269 140411597268160 submission.py:307] 56000) loss = 1.030, grad_norm = 0.500 +I0912 13:39:49.577824 140389990213376 logging_writer.py:48] [56500] global_step=56500, grad_norm=0.499999, loss=0.950858 +I0912 13:39:49.587805 140411597268160 submission.py:307] 56500) loss = 0.951, grad_norm = 0.500 +I0912 13:43:51.006364 140389973427968 logging_writer.py:48] [57000] global_step=57000, grad_norm=0.5, loss=0.96523 +I0912 13:43:51.010324 140411597268160 submission.py:307] 57000) loss = 0.965, grad_norm = 0.500 +I0912 13:46:28.971043 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 13:47:15.429677 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 13:48:31.763104 140411597268160 spec.py:363] Evaluating on the test split. +I0912 13:48:32.656969 140411597268160 submission_runner.py:516] Time since start: 40536.13s, Step: 57207, {'train/accuracy': 0.7190489477040817, 'train/loss': 4.503145178970025, 'validation/accuracy': 0.60846, 'validation/loss': 3.6531325, 'validation/num_examples': 50000, 'test/accuracy': 0.5381, 'test/loss': 2.3990828125, 'test/num_examples': 10000, 'score': 38026.40539455414, 'total_duration': 40536.130140542984, 'accumulated_submission_time': 38026.40539455414, 'accumulated_eval_time': 2414.7549934387207, 'accumulated_logging_time': 4.314009189605713} +I0912 13:48:32.889740 140387201582848 logging_writer.py:48] [57207] accumulated_eval_time=2414.75, accumulated_logging_time=4.31401, accumulated_submission_time=38026.4, global_step=57207, preemption_count=0, score=38026.4, test/accuracy=0.5381, test/loss=2.39908, test/num_examples=10000, total_duration=40536.1, train/accuracy=0.719049, train/loss=4.50315, validation/accuracy=0.60846, validation/loss=3.65313, validation/num_examples=50000 +I0912 13:52:23.445089 140390074140416 logging_writer.py:48] [57500] global_step=57500, grad_norm=0.5, loss=0.96894 +I0912 13:52:23.448817 140411597268160 submission.py:307] 57500) loss = 0.969, grad_norm = 0.500 +I0912 13:56:28.221646 140387201582848 logging_writer.py:48] [58000] global_step=58000, grad_norm=0.5, loss=0.989908 +I0912 13:56:28.225420 140411597268160 submission.py:307] 58000) loss = 0.990, grad_norm = 0.500 +I0912 14:02:36.311983 140390074140416 logging_writer.py:48] [58500] global_step=58500, grad_norm=0.5, loss=0.978405 +I0912 14:02:36.315675 140411597268160 submission.py:307] 58500) loss = 0.978, grad_norm = 0.500 +I0912 14:08:27.124010 140387201582848 logging_writer.py:48] [59000] global_step=59000, grad_norm=0.5, loss=0.965366 +I0912 14:08:27.128525 140411597268160 submission.py:307] 59000) loss = 0.965, grad_norm = 0.500 +I0912 14:12:29.719625 140390074140416 logging_writer.py:48] [59500] global_step=59500, grad_norm=0.5, loss=0.989736 +I0912 14:12:29.723464 140411597268160 submission.py:307] 59500) loss = 0.990, grad_norm = 0.500 +I0912 14:20:12.757153 140387201582848 logging_writer.py:48] [60000] global_step=60000, grad_norm=0.5, loss=1.02305 +I0912 14:20:12.761043 140411597268160 submission.py:307] 60000) loss = 1.023, grad_norm = 0.500 +I0912 14:21:51.121289 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 14:22:27.348691 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 14:23:30.747842 140411597268160 spec.py:363] Evaluating on the test split. +I0912 14:23:31.645980 140411597268160 submission_runner.py:516] Time since start: 42635.12s, Step: 60240, {'train/accuracy': 0.7339365433673469, 'train/loss': 4.250097780811544, 'validation/accuracy': 0.60516, 'validation/loss': 3.680009375, 'validation/num_examples': 50000, 'test/accuracy': 0.5419, 'test/loss': 2.3658654296875, 'test/num_examples': 10000, 'score': 40019.40243244171, 'total_duration': 42635.11920881271, 'accumulated_submission_time': 40019.40243244171, 'accumulated_eval_time': 2515.279800891876, 'accumulated_logging_time': 4.556050777435303} +I0912 14:23:31.907611 140389948249856 logging_writer.py:48] [60240] accumulated_eval_time=2515.28, accumulated_logging_time=4.55605, accumulated_submission_time=40019.4, global_step=60240, preemption_count=0, score=40019.4, test/accuracy=0.5419, test/loss=2.36587, test/num_examples=10000, total_duration=42635.1, train/accuracy=0.733937, train/loss=4.2501, validation/accuracy=0.60516, validation/loss=3.68001, validation/num_examples=50000 +I0912 14:25:38.272238 140390065747712 logging_writer.py:48] [60500] global_step=60500, grad_norm=0.5, loss=0.980697 +I0912 14:25:38.276071 140411597268160 submission.py:307] 60500) loss = 0.981, grad_norm = 0.500 +I0912 14:32:03.568670 140389948249856 logging_writer.py:48] [61000] global_step=61000, grad_norm=0.5, loss=1.00782 +I0912 14:32:03.572427 140411597268160 submission.py:307] 61000) loss = 1.008, grad_norm = 0.500 +I0912 14:38:12.435132 140390065747712 logging_writer.py:48] [61500] global_step=61500, grad_norm=0.499999, loss=0.893844 +I0912 14:38:12.440210 140411597268160 submission.py:307] 61500) loss = 0.894, grad_norm = 0.500 +I0912 14:42:07.450584 140389948249856 logging_writer.py:48] [62000] global_step=62000, grad_norm=0.5, loss=0.999146 +I0912 14:42:07.454456 140411597268160 submission.py:307] 62000) loss = 0.999, grad_norm = 0.500 +I0912 14:49:51.564681 140390065747712 logging_writer.py:48] [62500] global_step=62500, grad_norm=0.5, loss=0.926066 +I0912 14:49:51.568420 140411597268160 submission.py:307] 62500) loss = 0.926, grad_norm = 0.500 +I0912 14:53:19.429129 140389948249856 logging_writer.py:48] [63000] global_step=63000, grad_norm=0.5, loss=0.98849 +I0912 14:53:19.433715 140411597268160 submission.py:307] 63000) loss = 0.988, grad_norm = 0.500 +I0912 14:56:55.668468 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 14:57:41.647223 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 14:58:52.168135 140411597268160 spec.py:363] Evaluating on the test split. +I0912 14:58:53.030497 140411597268160 submission_runner.py:516] Time since start: 44756.50s, Step: 63355, {'train/accuracy': 0.7662228954081632, 'train/loss': 3.4690078423947703, 'validation/accuracy': 0.60764, 'validation/loss': 3.729314375, 'validation/num_examples': 50000, 'test/accuracy': 0.5399, 'test/loss': 2.441900390625, 'test/num_examples': 10000, 'score': 42017.28895163536, 'total_duration': 44756.503669023514, 'accumulated_submission_time': 42017.28895163536, 'accumulated_eval_time': 2632.6418223381042, 'accumulated_logging_time': 4.827040195465088} +I0912 14:58:53.253135 140390057355008 logging_writer.py:48] [63355] accumulated_eval_time=2632.64, accumulated_logging_time=4.82704, accumulated_submission_time=42017.3, global_step=63355, preemption_count=0, score=42017.3, test/accuracy=0.5399, test/loss=2.4419, test/num_examples=10000, total_duration=44756.5, train/accuracy=0.766223, train/loss=3.46901, validation/accuracy=0.60764, validation/loss=3.72931, validation/num_examples=50000 +I0912 15:00:13.164971 140389990213376 logging_writer.py:48] [63500] global_step=63500, grad_norm=0.5, loss=0.869769 +I0912 15:00:13.168719 140411597268160 submission.py:307] 63500) loss = 0.870, grad_norm = 0.500 +I0912 15:06:28.748045 140390057355008 logging_writer.py:48] [64000] global_step=64000, grad_norm=0.5, loss=0.828772 +I0912 15:06:28.752855 140411597268160 submission.py:307] 64000) loss = 0.829, grad_norm = 0.500 +I0912 15:10:59.171285 140389990213376 logging_writer.py:48] [64500] global_step=64500, grad_norm=0.5, loss=0.901852 +I0912 15:10:59.175330 140411597268160 submission.py:307] 64500) loss = 0.902, grad_norm = 0.500 +I0912 15:18:35.111548 140390057355008 logging_writer.py:48] [65000] global_step=65000, grad_norm=0.5, loss=0.96482 +I0912 15:18:35.115343 140411597268160 submission.py:307] 65000) loss = 0.965, grad_norm = 0.500 +I0912 15:21:59.742727 140389990213376 logging_writer.py:48] [65500] global_step=65500, grad_norm=0.5, loss=0.857888 +I0912 15:21:59.747727 140411597268160 submission.py:307] 65500) loss = 0.858, grad_norm = 0.500 +I0912 15:27:45.841111 140390057355008 logging_writer.py:48] [66000] global_step=66000, grad_norm=0.5, loss=0.938355 +I0912 15:27:45.844874 140411597268160 submission.py:307] 66000) loss = 0.938, grad_norm = 0.500 +I0912 15:32:11.306126 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 15:32:35.599513 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 15:33:39.217180 140411597268160 spec.py:363] Evaluating on the test split. +I0912 15:33:40.108738 140411597268160 submission_runner.py:516] Time since start: 46843.58s, Step: 66260, {'train/accuracy': 0.7411910076530612, 'train/loss': 4.541734967912946, 'validation/accuracy': 0.61386, 'validation/loss': 3.7965825, 'validation/num_examples': 50000, 'test/accuracy': 0.5465, 'test/loss': 2.424560546875, 'test/num_examples': 10000, 'score': 44011.472423791885, 'total_duration': 46843.58191680908, 'accumulated_submission_time': 44011.472423791885, 'accumulated_eval_time': 2721.444454908371, 'accumulated_logging_time': 5.059232711791992} +I0912 15:33:40.369746 140389956642560 logging_writer.py:48] [66260] accumulated_eval_time=2721.44, accumulated_logging_time=5.05923, accumulated_submission_time=44011.5, global_step=66260, preemption_count=0, score=44011.5, test/accuracy=0.5465, test/loss=2.42456, test/num_examples=10000, total_duration=46843.6, train/accuracy=0.741191, train/loss=4.54173, validation/accuracy=0.61386, validation/loss=3.79658, validation/num_examples=50000 +I0912 15:35:36.918537 140390074140416 logging_writer.py:48] [66500] global_step=66500, grad_norm=0.5, loss=0.836715 +I0912 15:35:36.922784 140411597268160 submission.py:307] 66500) loss = 0.837, grad_norm = 0.500 +I0912 15:40:21.909274 140389956642560 logging_writer.py:48] [67000] global_step=67000, grad_norm=0.5, loss=0.878646 +I0912 15:40:21.913088 140411597268160 submission.py:307] 67000) loss = 0.879, grad_norm = 0.500 +I0912 15:48:11.632773 140390074140416 logging_writer.py:48] [67500] global_step=67500, grad_norm=0.5, loss=0.867609 +I0912 15:48:11.636595 140411597268160 submission.py:307] 67500) loss = 0.868, grad_norm = 0.500 +I0912 15:51:41.039981 140389956642560 logging_writer.py:48] [68000] global_step=68000, grad_norm=0.5, loss=0.898535 +I0912 15:51:41.044165 140411597268160 submission.py:307] 68000) loss = 0.899, grad_norm = 0.500 +I0912 15:57:34.580519 140390074140416 logging_writer.py:48] [68500] global_step=68500, grad_norm=0.5, loss=0.954628 +I0912 15:57:34.584280 140411597268160 submission.py:307] 68500) loss = 0.955, grad_norm = 0.500 +I0912 16:03:35.914745 140389956642560 logging_writer.py:48] [69000] global_step=69000, grad_norm=0.5, loss=0.889634 +I0912 16:03:35.921107 140411597268160 submission.py:307] 69000) loss = 0.890, grad_norm = 0.500 +I0912 16:06:58.850305 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 16:07:52.106948 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 16:09:00.997643 140411597268160 spec.py:363] Evaluating on the test split. +I0912 16:09:01.857810 140411597268160 submission_runner.py:516] Time since start: 48965.33s, Step: 69453, {'train/accuracy': 0.7394571109693877, 'train/loss': 4.600998236208546, 'validation/accuracy': 0.60794, 'validation/loss': 3.766720625, 'validation/num_examples': 50000, 'test/accuracy': 0.5419, 'test/loss': 2.48736328125, 'test/num_examples': 10000, 'score': 46004.137469530106, 'total_duration': 48965.33098864555, 'accumulated_submission_time': 46004.137469530106, 'accumulated_eval_time': 2844.452032327652, 'accumulated_logging_time': 5.329614877700806} +I0912 16:09:02.165881 140387592111872 logging_writer.py:48] [69453] accumulated_eval_time=2844.45, accumulated_logging_time=5.32961, accumulated_submission_time=46004.1, global_step=69453, preemption_count=0, score=46004.1, test/accuracy=0.5419, test/loss=2.48736, test/num_examples=10000, total_duration=48965.3, train/accuracy=0.739457, train/loss=4.601, validation/accuracy=0.60794, validation/loss=3.76672, validation/num_examples=50000 +I0912 16:09:14.886339 140390099318528 logging_writer.py:48] [69500] global_step=69500, grad_norm=0.5, loss=0.900206 +I0912 16:09:14.889638 140411597268160 submission.py:307] 69500) loss = 0.900, grad_norm = 0.500 +I0912 16:16:54.347544 140387592111872 logging_writer.py:48] [70000] global_step=70000, grad_norm=0.5, loss=0.838184 +I0912 16:16:54.351400 140411597268160 submission.py:307] 70000) loss = 0.838, grad_norm = 0.500 +I0912 16:20:56.077470 140390099318528 logging_writer.py:48] [70500] global_step=70500, grad_norm=0.5, loss=0.842395 +I0912 16:20:56.082274 140411597268160 submission.py:307] 70500) loss = 0.842, grad_norm = 0.500 +I0912 16:26:44.143733 140387592111872 logging_writer.py:48] [71000] global_step=71000, grad_norm=0.5, loss=0.79205 +I0912 16:26:44.147678 140411597268160 submission.py:307] 71000) loss = 0.792, grad_norm = 0.500 +I0912 16:32:44.129396 140390099318528 logging_writer.py:48] [71500] global_step=71500, grad_norm=0.5, loss=0.818705 +I0912 16:32:44.135083 140411597268160 submission.py:307] 71500) loss = 0.819, grad_norm = 0.500 +I0912 16:36:38.243186 140387592111872 logging_writer.py:48] [72000] global_step=72000, grad_norm=0.5, loss=0.921557 +I0912 16:36:38.247169 140411597268160 submission.py:307] 72000) loss = 0.922, grad_norm = 0.500 +I0912 16:42:20.253082 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 16:42:54.063579 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 16:44:20.689672 140411597268160 spec.py:363] Evaluating on the test split. +I0912 16:44:21.551105 140411597268160 submission_runner.py:516] Time since start: 51085.02s, Step: 72390, {'train/accuracy': 0.7686344068877551, 'train/loss': 4.249235192123725, 'validation/accuracy': 0.61002, 'validation/loss': 3.8339628125, 'validation/num_examples': 50000, 'test/accuracy': 0.5405, 'test/loss': 2.52101484375, 'test/num_examples': 10000, 'score': 47997.16594219208, 'total_duration': 51085.02428627014, 'accumulated_submission_time': 47997.16594219208, 'accumulated_eval_time': 2965.7500581741333, 'accumulated_logging_time': 5.646993637084961} +I0912 16:44:21.797071 140390006998784 logging_writer.py:48] [72390] accumulated_eval_time=2965.75, accumulated_logging_time=5.64699, accumulated_submission_time=47997.2, global_step=72390, preemption_count=0, score=47997.2, test/accuracy=0.5405, test/loss=2.52101, test/num_examples=10000, total_duration=51085, train/accuracy=0.768634, train/loss=4.24924, validation/accuracy=0.61002, validation/loss=3.83396, validation/num_examples=50000 +I0912 16:45:08.882872 140390048962304 logging_writer.py:48] [72500] global_step=72500, grad_norm=0.5, loss=0.916 +I0912 16:45:08.886598 140411597268160 submission.py:307] 72500) loss = 0.916, grad_norm = 0.500 +I0912 16:49:27.241153 140390006998784 logging_writer.py:48] [73000] global_step=73000, grad_norm=0.5, loss=0.886878 +I0912 16:49:27.245145 140411597268160 submission.py:307] 73000) loss = 0.887, grad_norm = 0.500 +I0912 16:55:42.412860 140390048962304 logging_writer.py:48] [73500] global_step=73500, grad_norm=0.5, loss=0.912614 +I0912 16:55:42.417026 140411597268160 submission.py:307] 73500) loss = 0.913, grad_norm = 0.500 +I0912 17:01:38.306741 140390006998784 logging_writer.py:48] [74000] global_step=74000, grad_norm=0.5, loss=0.899296 +I0912 17:01:38.311408 140411597268160 submission.py:307] 74000) loss = 0.899, grad_norm = 0.500 +I0912 17:05:30.164259 140390048962304 logging_writer.py:48] [74500] global_step=74500, grad_norm=0.5, loss=0.911122 +I0912 17:05:30.168349 140411597268160 submission.py:307] 74500) loss = 0.911, grad_norm = 0.500 +I0912 17:13:17.559234 140390006998784 logging_writer.py:48] [75000] global_step=75000, grad_norm=0.5, loss=0.832376 +I0912 17:13:17.563047 140411597268160 submission.py:307] 75000) loss = 0.832, grad_norm = 0.500 +I0912 17:16:46.970399 140390048962304 logging_writer.py:48] [75500] global_step=75500, grad_norm=0.5, loss=0.842876 +I0912 17:16:46.974355 140411597268160 submission.py:307] 75500) loss = 0.843, grad_norm = 0.500 +I0912 17:17:40.244601 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 17:18:26.533801 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 17:19:31.502663 140411597268160 spec.py:363] Evaluating on the test split. +I0912 17:19:32.363318 140411597268160 submission_runner.py:516] Time since start: 53195.84s, Step: 75606, {'train/accuracy': 0.7893813775510204, 'train/loss': 3.65380859375, 'validation/accuracy': 0.60568, 'validation/loss': 3.93482125, 'validation/num_examples': 50000, 'test/accuracy': 0.5424, 'test/loss': 2.534219921875, 'test/num_examples': 10000, 'score': 49989.45276546478, 'total_duration': 53195.83649134636, 'accumulated_submission_time': 49989.45276546478, 'accumulated_eval_time': 3077.8687732219696, 'accumulated_logging_time': 5.9023377895355225} +I0912 17:19:32.633461 140390006998784 logging_writer.py:48] [75606] accumulated_eval_time=3077.87, accumulated_logging_time=5.90234, accumulated_submission_time=49989.5, global_step=75606, preemption_count=0, score=49989.5, test/accuracy=0.5424, test/loss=2.53422, test/num_examples=10000, total_duration=53195.8, train/accuracy=0.789381, train/loss=3.65381, validation/accuracy=0.60568, validation/loss=3.93482, validation/num_examples=50000 +I0912 17:24:34.267189 140389981820672 logging_writer.py:48] [76000] global_step=76000, grad_norm=0.5, loss=0.822129 +I0912 17:24:34.271057 140411597268160 submission.py:307] 76000) loss = 0.822, grad_norm = 0.500 +I0912 17:30:59.714081 140390006998784 logging_writer.py:48] [76500] global_step=76500, grad_norm=0.5, loss=0.814313 +I0912 17:30:59.718319 140411597268160 submission.py:307] 76500) loss = 0.814, grad_norm = 0.500 +I0912 17:34:55.771270 140389981820672 logging_writer.py:48] [77000] global_step=77000, grad_norm=0.5, loss=0.843721 +I0912 17:34:55.774919 140411597268160 submission.py:307] 77000) loss = 0.844, grad_norm = 0.500 +I0912 17:42:39.764246 140390006998784 logging_writer.py:48] [77500] global_step=77500, grad_norm=0.5, loss=0.771653 +I0912 17:42:39.767951 140411597268160 submission.py:307] 77500) loss = 0.772, grad_norm = 0.500 +I0912 17:46:09.614822 140389981820672 logging_writer.py:48] [78000] global_step=78000, grad_norm=0.5, loss=0.801329 +I0912 17:46:09.618901 140411597268160 submission.py:307] 78000) loss = 0.801, grad_norm = 0.500 +I0912 17:51:33.375078 140390006998784 logging_writer.py:48] [78500] global_step=78500, grad_norm=0.5, loss=0.860907 +I0912 17:51:33.378917 140411597268160 submission.py:307] 78500) loss = 0.861, grad_norm = 0.500 +I0912 17:52:50.067901 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 17:53:27.471028 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 17:54:47.410161 140411597268160 spec.py:363] Evaluating on the test split. +I0912 17:54:48.271432 140411597268160 submission_runner.py:516] Time since start: 55311.74s, Step: 78568, {'train/accuracy': 0.8084143813775511, 'train/loss': 3.4343616719148597, 'validation/accuracy': 0.60452, 'validation/loss': 4.179504375, 'validation/num_examples': 50000, 'test/accuracy': 0.5439, 'test/loss': 2.54808125, 'test/num_examples': 10000, 'score': 51982.49979019165, 'total_duration': 55311.744655132294, 'accumulated_submission_time': 51982.49979019165, 'accumulated_eval_time': 3196.0724246501923, 'accumulated_logging_time': 6.18151068687439} +I0912 17:54:48.650378 140390082533120 logging_writer.py:48] [78568] accumulated_eval_time=3196.07, accumulated_logging_time=6.18151, accumulated_submission_time=51982.5, global_step=78568, preemption_count=0, score=51982.5, test/accuracy=0.5439, test/loss=2.54808, test/num_examples=10000, total_duration=55311.7, train/accuracy=0.808414, train/loss=3.43436, validation/accuracy=0.60452, validation/loss=4.1795, validation/num_examples=50000 +I0912 17:59:07.265183 140389973427968 logging_writer.py:48] [79000] global_step=79000, grad_norm=0.5, loss=0.743133 +I0912 17:59:07.269454 140411597268160 submission.py:307] 79000) loss = 0.743, grad_norm = 0.500 +I0912 18:03:38.991789 140390082533120 logging_writer.py:48] [79500] global_step=79500, grad_norm=0.5, loss=0.810817 +I0912 18:03:38.995506 140411597268160 submission.py:307] 79500) loss = 0.811, grad_norm = 0.500 +I0912 18:11:12.327372 140389973427968 logging_writer.py:48] [80000] global_step=80000, grad_norm=0.5, loss=0.878504 +I0912 18:11:12.331257 140411597268160 submission.py:307] 80000) loss = 0.879, grad_norm = 0.500 +I0912 18:14:41.537234 140390082533120 logging_writer.py:48] [80500] global_step=80500, grad_norm=0.5, loss=0.846268 +I0912 18:14:41.542546 140411597268160 submission.py:307] 80500) loss = 0.846, grad_norm = 0.500 +I0912 18:20:09.721250 140389973427968 logging_writer.py:48] [81000] global_step=81000, grad_norm=0.5, loss=0.793304 +I0912 18:20:09.724951 140411597268160 submission.py:307] 81000) loss = 0.793, grad_norm = 0.500 +I0912 18:26:29.179116 140390082533120 logging_writer.py:48] [81500] global_step=81500, grad_norm=0.5, loss=0.726836 +I0912 18:26:29.201896 140411597268160 submission.py:307] 81500) loss = 0.727, grad_norm = 0.500 +I0912 18:28:06.737611 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 18:28:52.440334 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 18:29:56.453804 140411597268160 spec.py:363] Evaluating on the test split. +I0912 18:29:57.314420 140411597268160 submission_runner.py:516] Time since start: 57420.79s, Step: 81755, {'train/accuracy': 0.7738360969387755, 'train/loss': 4.3157329948581, 'validation/accuracy': 0.60878, 'validation/loss': 3.9008559375, 'validation/num_examples': 50000, 'test/accuracy': 0.5395, 'test/loss': 2.718340625, 'test/num_examples': 10000, 'score': 53974.51514530182, 'total_duration': 57420.78761053085, 'accumulated_submission_time': 53974.51514530182, 'accumulated_eval_time': 3306.6501145362854, 'accumulated_logging_time': 6.569916725158691} +I0912 18:29:57.578937 140389981820672 logging_writer.py:48] [81755] accumulated_eval_time=3306.65, accumulated_logging_time=6.56992, accumulated_submission_time=53974.5, global_step=81755, preemption_count=0, score=53974.5, test/accuracy=0.5395, test/loss=2.71834, test/num_examples=10000, total_duration=57420.8, train/accuracy=0.773836, train/loss=4.31573, validation/accuracy=0.60878, validation/loss=3.90086, validation/num_examples=50000 +I0912 18:32:22.610202 140390032176896 logging_writer.py:48] [82000] global_step=82000, grad_norm=0.5, loss=0.772175 +I0912 18:32:22.613736 140411597268160 submission.py:307] 82000) loss = 0.772, grad_norm = 0.500 +I0912 18:40:15.254025 140389981820672 logging_writer.py:48] [82500] global_step=82500, grad_norm=0.499999, loss=0.854074 +I0912 18:40:15.257753 140411597268160 submission.py:307] 82500) loss = 0.854, grad_norm = 0.500 +I0912 18:44:06.837411 140390032176896 logging_writer.py:48] [83000] global_step=83000, grad_norm=0.5, loss=0.811201 +I0912 18:44:06.841682 140411597268160 submission.py:307] 83000) loss = 0.811, grad_norm = 0.500 +I0912 18:49:39.439405 140389981820672 logging_writer.py:48] [83500] global_step=83500, grad_norm=0.5, loss=0.816516 +I0912 18:49:39.443061 140411597268160 submission.py:307] 83500) loss = 0.817, grad_norm = 0.500 +I0912 18:55:57.647045 140390032176896 logging_writer.py:48] [84000] global_step=84000, grad_norm=0.5, loss=0.758524 +I0912 18:55:57.652825 140411597268160 submission.py:307] 84000) loss = 0.759, grad_norm = 0.500 +I0912 18:59:48.859996 140389981820672 logging_writer.py:48] [84500] global_step=84500, grad_norm=0.5, loss=0.846753 +I0912 18:59:48.863786 140411597268160 submission.py:307] 84500) loss = 0.847, grad_norm = 0.500 +I0912 19:03:15.488720 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 19:03:56.898503 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 19:05:18.848312 140411597268160 spec.py:363] Evaluating on the test split. +I0912 19:05:19.709894 140411597268160 submission_runner.py:516] Time since start: 59543.18s, Step: 84769, {'train/accuracy': 0.7844985650510204, 'train/loss': 4.187828219666773, 'validation/accuracy': 0.5956, 'validation/loss': 4.42532375, 'validation/num_examples': 50000, 'test/accuracy': 0.5363, 'test/loss': 2.6479666015625, 'test/num_examples': 10000, 'score': 55967.39384794235, 'total_duration': 59543.1830201149, 'accumulated_submission_time': 55967.39384794235, 'accumulated_eval_time': 3430.871250152588, 'accumulated_logging_time': 6.843689918518066} +I0912 19:05:20.037100 140390090925824 logging_writer.py:48] [84769] accumulated_eval_time=3430.87, accumulated_logging_time=6.84369, accumulated_submission_time=55967.4, global_step=84769, preemption_count=0, score=55967.4, test/accuracy=0.5363, test/loss=2.64797, test/num_examples=10000, total_duration=59543.2, train/accuracy=0.784499, train/loss=4.18783, validation/accuracy=0.5956, validation/loss=4.42532, validation/num_examples=50000 +I0912 19:08:09.541769 140390015391488 logging_writer.py:48] [85000] global_step=85000, grad_norm=0.5, loss=0.738188 +I0912 19:08:09.545428 140411597268160 submission.py:307] 85000) loss = 0.738, grad_norm = 0.500 +I0912 19:12:23.777981 140390090925824 logging_writer.py:48] [85500] global_step=85500, grad_norm=0.5, loss=0.801883 +I0912 19:12:23.782583 140411597268160 submission.py:307] 85500) loss = 0.802, grad_norm = 0.500 +I0912 19:18:26.098675 140390015391488 logging_writer.py:48] [86000] global_step=86000, grad_norm=0.5, loss=0.769278 +I0912 19:18:26.102424 140411597268160 submission.py:307] 86000) loss = 0.769, grad_norm = 0.500 +I0912 19:24:30.399685 140390090925824 logging_writer.py:48] [86500] global_step=86500, grad_norm=0.5, loss=0.754439 +I0912 19:24:30.407392 140411597268160 submission.py:307] 86500) loss = 0.754, grad_norm = 0.500 +I0912 19:28:20.195382 140390015391488 logging_writer.py:48] [87000] global_step=87000, grad_norm=0.5, loss=0.73879 +I0912 19:28:20.199224 140411597268160 submission.py:307] 87000) loss = 0.739, grad_norm = 0.500 +I0912 19:35:59.039781 140390090925824 logging_writer.py:48] [87500] global_step=87500, grad_norm=0.5, loss=0.775342 +I0912 19:35:59.043557 140411597268160 submission.py:307] 87500) loss = 0.775, grad_norm = 0.500 +I0912 19:38:37.612382 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 19:39:15.221458 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 19:40:19.317577 140411597268160 spec.py:363] Evaluating on the test split. +I0912 19:40:20.209088 140411597268160 submission_runner.py:516] Time since start: 61643.68s, Step: 87863, {'train/accuracy': 0.8123604910714286, 'train/loss': 3.558631741270727, 'validation/accuracy': 0.60924, 'validation/loss': 4.02082625, 'validation/num_examples': 50000, 'test/accuracy': 0.5439, 'test/loss': 2.617235546875, 'test/num_examples': 10000, 'score': 57960.61058306694, 'total_duration': 61643.68226647377, 'accumulated_submission_time': 57960.61058306694, 'accumulated_eval_time': 3533.468069791794, 'accumulated_logging_time': 7.181390047073364} +I0912 19:40:20.575733 140389973427968 logging_writer.py:48] [87863] accumulated_eval_time=3533.47, accumulated_logging_time=7.18139, accumulated_submission_time=57960.6, global_step=87863, preemption_count=0, score=57960.6, test/accuracy=0.5439, test/loss=2.61724, test/num_examples=10000, total_duration=61643.7, train/accuracy=0.81236, train/loss=3.55863, validation/accuracy=0.60924, validation/loss=4.02083, validation/num_examples=50000 +I0912 19:41:11.181840 140389981820672 logging_writer.py:48] [88000] global_step=88000, grad_norm=0.5, loss=0.771352 +I0912 19:41:11.186013 140411597268160 submission.py:307] 88000) loss = 0.771, grad_norm = 0.500 +I0912 19:47:45.553315 140389973427968 logging_writer.py:48] [88500] global_step=88500, grad_norm=0.5, loss=0.789712 +I0912 19:47:45.557054 140411597268160 submission.py:307] 88500) loss = 0.790, grad_norm = 0.500 +I0912 19:54:15.370921 140389981820672 logging_writer.py:48] [89000] global_step=89000, grad_norm=0.5, loss=0.712802 +I0912 19:54:15.390014 140411597268160 submission.py:307] 89000) loss = 0.713, grad_norm = 0.500 +I0912 19:57:58.570518 140389973427968 logging_writer.py:48] [89500] global_step=89500, grad_norm=0.5, loss=0.664049 +I0912 19:57:58.574433 140411597268160 submission.py:307] 89500) loss = 0.664, grad_norm = 0.500 +I0912 20:05:40.286255 140389981820672 logging_writer.py:48] [90000] global_step=90000, grad_norm=0.5, loss=0.784235 +I0912 20:05:40.290140 140411597268160 submission.py:307] 90000) loss = 0.784, grad_norm = 0.500 +I0912 20:09:19.642434 140389973427968 logging_writer.py:48] [90500] global_step=90500, grad_norm=0.5, loss=0.844771 +I0912 20:09:19.653460 140411597268160 submission.py:307] 90500) loss = 0.845, grad_norm = 0.500 +I0912 20:13:37.863559 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 20:14:26.049382 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 20:15:44.211918 140411597268160 spec.py:363] Evaluating on the test split. +I0912 20:15:45.111709 140411597268160 submission_runner.py:516] Time since start: 63768.58s, Step: 90929, {'train/accuracy': 0.8195352359693877, 'train/loss': 3.6061793736049106, 'validation/accuracy': 0.60892, 'validation/loss': 4.214265, 'validation/num_examples': 50000, 'test/accuracy': 0.5419, 'test/loss': 2.7006576171875, 'test/num_examples': 10000, 'score': 59953.10913348198, 'total_duration': 63768.584854602814, 'accumulated_submission_time': 59953.10913348198, 'accumulated_eval_time': 3660.7162594795227, 'accumulated_logging_time': 7.5574729442596436} +I0912 20:15:45.383143 140389981820672 logging_writer.py:48] [90929] accumulated_eval_time=3660.72, accumulated_logging_time=7.55747, accumulated_submission_time=59953.1, global_step=90929, preemption_count=0, score=59953.1, test/accuracy=0.5419, test/loss=2.70066, test/num_examples=10000, total_duration=63768.6, train/accuracy=0.819535, train/loss=3.60618, validation/accuracy=0.60892, validation/loss=4.21427, validation/num_examples=50000 +I0912 20:16:04.532231 140390032176896 logging_writer.py:48] [91000] global_step=91000, grad_norm=0.5, loss=0.747529 +I0912 20:16:04.536923 140411597268160 submission.py:307] 91000) loss = 0.748, grad_norm = 0.500 +I0912 20:22:23.753712 140389981820672 logging_writer.py:48] [91500] global_step=91500, grad_norm=0.5, loss=0.776157 +I0912 20:22:23.779552 140411597268160 submission.py:307] 91500) loss = 0.776, grad_norm = 0.500 +I0912 20:26:51.011304 140390032176896 logging_writer.py:48] [92000] global_step=92000, grad_norm=0.5, loss=0.663301 +I0912 20:26:51.015775 140411597268160 submission.py:307] 92000) loss = 0.663, grad_norm = 0.500 +I0912 20:34:15.056970 140389981820672 logging_writer.py:48] [92500] global_step=92500, grad_norm=0.5, loss=0.73193 +I0912 20:34:15.060732 140411597268160 submission.py:307] 92500) loss = 0.732, grad_norm = 0.500 +I0912 20:37:55.642435 140390032176896 logging_writer.py:48] [93000] global_step=93000, grad_norm=0.5, loss=0.739122 +I0912 20:37:55.646772 140411597268160 submission.py:307] 93000) loss = 0.739, grad_norm = 0.500 +I0912 20:43:24.937524 140389981820672 logging_writer.py:48] [93500] global_step=93500, grad_norm=0.5, loss=0.718059 +I0912 20:43:24.941393 140411597268160 submission.py:307] 93500) loss = 0.718, grad_norm = 0.500 +I0912 20:49:05.089445 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 20:49:39.025316 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 20:50:42.332840 140411597268160 spec.py:363] Evaluating on the test split. +I0912 20:50:43.243312 140411597268160 submission_runner.py:516] Time since start: 65866.72s, Step: 93885, {'train/accuracy': 0.7375438456632653, 'train/loss': 6.002884845344388, 'validation/accuracy': 0.61222, 'validation/loss': 4.09152, 'validation/num_examples': 50000, 'test/accuracy': 0.5458, 'test/loss': 2.68063203125, 'test/num_examples': 10000, 'score': 61947.415028095245, 'total_duration': 65866.71648287773, 'accumulated_submission_time': 61947.415028095245, 'accumulated_eval_time': 3758.8702669143677, 'accumulated_logging_time': 7.838247060775757} +I0912 20:50:43.524527 140390015391488 logging_writer.py:48] [93885] accumulated_eval_time=3758.87, accumulated_logging_time=7.83825, accumulated_submission_time=61947.4, global_step=93885, preemption_count=0, score=61947.4, test/accuracy=0.5458, test/loss=2.68063, test/num_examples=10000, total_duration=65866.7, train/accuracy=0.737544, train/loss=6.00288, validation/accuracy=0.61222, validation/loss=4.09152, validation/num_examples=50000 +I0912 20:51:13.475622 140390090925824 logging_writer.py:48] [94000] global_step=94000, grad_norm=0.499999, loss=0.633835 +I0912 20:51:13.479498 140411597268160 submission.py:307] 94000) loss = 0.634, grad_norm = 0.500 +I0912 20:56:17.357203 140390015391488 logging_writer.py:48] [94500] global_step=94500, grad_norm=0.5, loss=0.771777 +I0912 20:56:17.360852 140411597268160 submission.py:307] 94500) loss = 0.772, grad_norm = 0.500 +I0912 21:04:07.920850 140390090925824 logging_writer.py:48] [95000] global_step=95000, grad_norm=0.5, loss=0.741306 +I0912 21:04:07.924740 140411597268160 submission.py:307] 95000) loss = 0.741, grad_norm = 0.500 +I0912 21:07:47.700174 140390015391488 logging_writer.py:48] [95500] global_step=95500, grad_norm=0.5, loss=0.69903 +I0912 21:07:47.704528 140411597268160 submission.py:307] 95500) loss = 0.699, grad_norm = 0.500 +I0912 21:13:27.182260 140390090925824 logging_writer.py:48] [96000] global_step=96000, grad_norm=0.5, loss=0.644758 +I0912 21:13:27.186219 140411597268160 submission.py:307] 96000) loss = 0.645, grad_norm = 0.500 +I0912 21:19:44.716487 140390015391488 logging_writer.py:48] [96500] global_step=96500, grad_norm=0.5, loss=0.700516 +I0912 21:19:44.745740 140411597268160 submission.py:307] 96500) loss = 0.701, grad_norm = 0.500 +I0912 21:23:32.569347 140390090925824 logging_writer.py:48] [97000] global_step=97000, grad_norm=0.5, loss=0.652286 +I0912 21:23:32.573282 140411597268160 submission.py:307] 97000) loss = 0.652, grad_norm = 0.500 +I0912 21:24:02.003875 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 21:24:48.429954 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 21:25:55.742979 140411597268160 spec.py:363] Evaluating on the test split. +I0912 21:25:56.604556 140411597268160 submission_runner.py:516] Time since start: 67980.08s, Step: 97041, {'train/accuracy': 0.7970145089285714, 'train/loss': 4.257398644272162, 'validation/accuracy': 0.60598, 'validation/loss': 4.1292421875, 'validation/num_examples': 50000, 'test/accuracy': 0.5379, 'test/loss': 2.7065611328125, 'test/num_examples': 10000, 'score': 63939.98579668999, 'total_duration': 67980.07772994041, 'accumulated_submission_time': 63939.98579668999, 'accumulated_eval_time': 3873.47101354599, 'accumulated_logging_time': 8.128909587860107} +I0912 21:25:56.983042 140389973427968 logging_writer.py:48] [97041] accumulated_eval_time=3873.47, accumulated_logging_time=8.12891, accumulated_submission_time=63940, global_step=97041, preemption_count=0, score=63940, test/accuracy=0.5379, test/loss=2.70656, test/num_examples=10000, total_duration=67980.1, train/accuracy=0.797015, train/loss=4.2574, validation/accuracy=0.60598, validation/loss=4.12924, validation/num_examples=50000 +I0912 21:32:16.640394 140390074140416 logging_writer.py:48] [97500] global_step=97500, grad_norm=0.5, loss=0.756342 +I0912 21:32:16.644330 140411597268160 submission.py:307] 97500) loss = 0.756, grad_norm = 0.500 +I0912 21:36:43.067152 140389973427968 logging_writer.py:48] [98000] global_step=98000, grad_norm=0.5, loss=0.670761 +I0912 21:36:43.071080 140411597268160 submission.py:307] 98000) loss = 0.671, grad_norm = 0.500 +I0912 21:42:27.130776 140390074140416 logging_writer.py:48] [98500] global_step=98500, grad_norm=0.5, loss=0.759739 +I0912 21:42:27.134737 140411597268160 submission.py:307] 98500) loss = 0.760, grad_norm = 0.500 +I0912 21:48:33.973226 140389973427968 logging_writer.py:48] [99000] global_step=99000, grad_norm=0.5, loss=0.638876 +I0912 21:48:33.983992 140411597268160 submission.py:307] 99000) loss = 0.639, grad_norm = 0.500 +I0912 21:52:21.624979 140390074140416 logging_writer.py:48] [99500] global_step=99500, grad_norm=0.5, loss=0.633305 +I0912 21:52:21.629065 140411597268160 submission.py:307] 99500) loss = 0.633, grad_norm = 0.500 +I0912 21:59:21.734226 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 21:59:54.362386 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 22:01:16.475771 140411597268160 spec.py:363] Evaluating on the test split. +I0912 22:01:17.340138 140411597268160 submission_runner.py:516] Time since start: 70100.81s, Step: 99969, {'train/accuracy': 0.7916533801020408, 'train/loss': 4.6550697793765945, 'validation/accuracy': 0.60732, 'validation/loss': 4.32330125, 'validation/num_examples': 50000, 'test/accuracy': 0.537, 'test/loss': 2.738551953125, 'test/num_examples': 10000, 'score': 65939.92808747292, 'total_duration': 70100.81326365471, 'accumulated_submission_time': 65939.92808747292, 'accumulated_eval_time': 3989.0768382549286, 'accumulated_logging_time': 8.517204523086548} +I0912 22:01:17.641635 140390090925824 logging_writer.py:48] [99969] accumulated_eval_time=3989.08, accumulated_logging_time=8.5172, accumulated_submission_time=65939.9, global_step=99969, preemption_count=0, score=65939.9, test/accuracy=0.537, test/loss=2.73855, test/num_examples=10000, total_duration=70100.8, train/accuracy=0.791653, train/loss=4.65507, validation/accuracy=0.60732, validation/loss=4.3233, validation/num_examples=50000 +I0912 22:01:26.448365 140389965035264 logging_writer.py:48] [100000] global_step=100000, grad_norm=0.5, loss=0.639177 +I0912 22:01:26.451858 140411597268160 submission.py:307] 100000) loss = 0.639, grad_norm = 0.500 +I0912 22:05:26.235613 140390090925824 logging_writer.py:48] [100500] global_step=100500, grad_norm=0.5, loss=0.665361 +I0912 22:05:26.239381 140411597268160 submission.py:307] 100500) loss = 0.665, grad_norm = 0.500 +I0912 22:11:35.435483 140389965035264 logging_writer.py:48] [101000] global_step=101000, grad_norm=0.5, loss=0.707853 +I0912 22:11:35.439387 140411597268160 submission.py:307] 101000) loss = 0.708, grad_norm = 0.500 +I0912 22:17:47.307735 140390090925824 logging_writer.py:48] [101500] global_step=101500, grad_norm=0.5, loss=0.632475 +I0912 22:17:47.333054 140411597268160 submission.py:307] 101500) loss = 0.632, grad_norm = 0.500 +I0912 22:21:37.281486 140389965035264 logging_writer.py:48] [102000] global_step=102000, grad_norm=0.5, loss=0.755266 +I0912 22:21:37.285415 140411597268160 submission.py:307] 102000) loss = 0.755, grad_norm = 0.500 +I0912 22:29:17.549704 140390090925824 logging_writer.py:48] [102500] global_step=102500, grad_norm=0.5, loss=0.700239 +I0912 22:29:17.553570 140411597268160 submission.py:307] 102500) loss = 0.700, grad_norm = 0.500 +I0912 22:32:51.801691 140389965035264 logging_writer.py:48] [103000] global_step=103000, grad_norm=0.5, loss=0.673738 +I0912 22:32:51.807988 140411597268160 submission.py:307] 103000) loss = 0.674, grad_norm = 0.500 +I0912 22:34:36.071446 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 22:35:27.349787 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 22:36:34.580155 140411597268160 spec.py:363] Evaluating on the test split. +I0912 22:36:35.442712 140411597268160 submission_runner.py:516] Time since start: 72218.92s, Step: 103202, {'train/accuracy': 0.8127391581632653, 'train/loss': 4.210806710379464, 'validation/accuracy': 0.61348, 'validation/loss': 4.36987, 'validation/num_examples': 50000, 'test/accuracy': 0.5385, 'test/loss': 2.759041015625, 'test/num_examples': 10000, 'score': 67932.3721754551, 'total_duration': 72218.91586017609, 'accumulated_submission_time': 67932.3721754551, 'accumulated_eval_time': 4108.448180913925, 'accumulated_logging_time': 8.829198837280273} +I0912 22:36:35.772975 140390099318528 logging_writer.py:48] [103202] accumulated_eval_time=4108.45, accumulated_logging_time=8.8292, accumulated_submission_time=67932.4, global_step=103202, preemption_count=0, score=67932.4, test/accuracy=0.5385, test/loss=2.75904, test/num_examples=10000, total_duration=72218.9, train/accuracy=0.812739, train/loss=4.21081, validation/accuracy=0.61348, validation/loss=4.36987, validation/num_examples=50000 +I0912 22:40:13.748826 140390040569600 logging_writer.py:48] [103500] global_step=103500, grad_norm=0.5, loss=0.704589 +I0912 22:40:13.753278 140411597268160 submission.py:307] 103500) loss = 0.705, grad_norm = 0.500 +I0912 22:46:49.848510 140390099318528 logging_writer.py:48] [104000] global_step=104000, grad_norm=0.5, loss=0.654107 +I0912 22:46:49.852579 140411597268160 submission.py:307] 104000) loss = 0.654, grad_norm = 0.500 +I0912 22:50:58.179289 140390040569600 logging_writer.py:48] [104500] global_step=104500, grad_norm=0.5, loss=0.650874 +I0912 22:50:58.183341 140411597268160 submission.py:307] 104500) loss = 0.651, grad_norm = 0.500 +I0912 22:58:29.806497 140390099318528 logging_writer.py:48] [105000] global_step=105000, grad_norm=0.5, loss=0.736743 +I0912 22:58:29.810378 140411597268160 submission.py:307] 105000) loss = 0.737, grad_norm = 0.500 +I0912 23:02:15.560140 140390040569600 logging_writer.py:48] [105500] global_step=105500, grad_norm=0.5, loss=0.697499 +I0912 23:02:15.564248 140411597268160 submission.py:307] 105500) loss = 0.697, grad_norm = 0.500 +I0912 23:07:33.144525 140390099318528 logging_writer.py:48] [106000] global_step=106000, grad_norm=0.5, loss=0.658913 +I0912 23:07:33.148342 140411597268160 submission.py:307] 106000) loss = 0.659, grad_norm = 0.500 +I0912 23:09:53.513531 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 23:10:28.987664 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 23:11:54.337498 140411597268160 spec.py:363] Evaluating on the test split. +I0912 23:11:55.215873 140411597268160 submission_runner.py:516] Time since start: 74338.69s, Step: 106137, {'train/accuracy': 0.8254145408163265, 'train/loss': 3.80894812759088, 'validation/accuracy': 0.60976, 'validation/loss': 4.3623375, 'validation/num_examples': 50000, 'test/accuracy': 0.5392, 'test/loss': 2.8169044921875, 'test/num_examples': 10000, 'score': 69925.38329291344, 'total_duration': 74338.68905472755, 'accumulated_submission_time': 69925.38329291344, 'accumulated_eval_time': 4230.150585651398, 'accumulated_logging_time': 9.16890573501587} +I0912 23:11:55.526952 140387201582848 logging_writer.py:48] [106137] accumulated_eval_time=4230.15, accumulated_logging_time=9.16891, accumulated_submission_time=69925.4, global_step=106137, preemption_count=0, score=69925.4, test/accuracy=0.5392, test/loss=2.8169, test/num_examples=10000, total_duration=74338.7, train/accuracy=0.825415, train/loss=3.80895, validation/accuracy=0.60976, validation/loss=4.36234, validation/num_examples=50000 +I0912 23:15:20.469683 140390023784192 logging_writer.py:48] [106500] global_step=106500, grad_norm=0.5, loss=0.547094 +I0912 23:15:20.473444 140411597268160 submission.py:307] 106500) loss = 0.547, grad_norm = 0.500 +I0912 23:19:53.510751 140387201582848 logging_writer.py:48] [107000] global_step=107000, grad_norm=0.5, loss=0.638036 +I0912 23:19:53.514527 140411597268160 submission.py:307] 107000) loss = 0.638, grad_norm = 0.500 +I0912 23:27:26.550165 140390023784192 logging_writer.py:48] [107500] global_step=107500, grad_norm=0.5, loss=0.69401 +I0912 23:27:26.554257 140411597268160 submission.py:307] 107500) loss = 0.694, grad_norm = 0.500 +I0912 23:31:10.385007 140387201582848 logging_writer.py:48] [108000] global_step=108000, grad_norm=0.499999, loss=0.607678 +I0912 23:31:10.389440 140411597268160 submission.py:307] 108000) loss = 0.608, grad_norm = 0.500 +I0912 23:36:47.712199 140390023784192 logging_writer.py:48] [108500] global_step=108500, grad_norm=0.5, loss=0.644092 +I0912 23:36:47.715909 140411597268160 submission.py:307] 108500) loss = 0.644, grad_norm = 0.500 +I0912 23:43:10.777082 140387201582848 logging_writer.py:48] [109000] global_step=109000, grad_norm=0.5, loss=0.676606 +I0912 23:43:10.806806 140411597268160 submission.py:307] 109000) loss = 0.677, grad_norm = 0.500 +I0912 23:45:13.158087 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 23:45:57.115120 140411597268160 spec.py:346] Evaluating on the validation split. +I0912 23:47:01.889712 140411597268160 spec.py:363] Evaluating on the test split. +I0912 23:47:02.886205 140411597268160 submission_runner.py:516] Time since start: 76446.36s, Step: 109305, {'train/accuracy': 0.8047472895408163, 'train/loss': 4.569540218431122, 'validation/accuracy': 0.6078, 'validation/loss': 4.565165, 'validation/num_examples': 50000, 'test/accuracy': 0.5423, 'test/loss': 2.781318359375, 'test/num_examples': 10000, 'score': 71917.54116845131, 'total_duration': 76446.3592095375, 'accumulated_submission_time': 71917.54116845131, 'accumulated_eval_time': 4339.878563404083, 'accumulated_logging_time': 9.489476919174194} +I0912 23:47:03.153156 140390074140416 logging_writer.py:48] [109305] accumulated_eval_time=4339.88, accumulated_logging_time=9.48948, accumulated_submission_time=71917.5, global_step=109305, preemption_count=0, score=71917.5, test/accuracy=0.5423, test/loss=2.78132, test/num_examples=10000, total_duration=76446.4, train/accuracy=0.804747, train/loss=4.56954, validation/accuracy=0.6078, validation/loss=4.56517, validation/num_examples=50000 +I0912 23:48:51.713505 140390065747712 logging_writer.py:48] [109500] global_step=109500, grad_norm=0.5, loss=0.589508 +I0912 23:48:51.717286 140411597268160 submission.py:307] 109500) loss = 0.590, grad_norm = 0.500 +I0912 23:56:41.332116 140390074140416 logging_writer.py:48] [110000] global_step=110000, grad_norm=0.5, loss=0.634296 +I0912 23:56:41.336154 140411597268160 submission.py:307] 110000) loss = 0.634, grad_norm = 0.500 +I0913 00:00:49.300246 140390065747712 logging_writer.py:48] [110500] global_step=110500, grad_norm=0.5, loss=0.653423 +I0913 00:00:49.305255 140411597268160 submission.py:307] 110500) loss = 0.653, grad_norm = 0.500 +I0913 00:06:09.733582 140390074140416 logging_writer.py:48] [111000] global_step=111000, grad_norm=0.5, loss=0.654046 +I0913 00:06:09.737373 140411597268160 submission.py:307] 111000) loss = 0.654, grad_norm = 0.500 +I0913 00:12:40.306806 140390065747712 logging_writer.py:48] [111500] global_step=111500, grad_norm=0.5, loss=0.549371 +I0913 00:12:40.324527 140411597268160 submission.py:307] 111500) loss = 0.549, grad_norm = 0.500 +I0913 00:16:31.726883 140390074140416 logging_writer.py:48] [112000] global_step=112000, grad_norm=0.5, loss=0.550006 +I0913 00:16:31.730891 140411597268160 submission.py:307] 112000) loss = 0.550, grad_norm = 0.500 +I0913 00:20:23.015877 140411597268160 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 00:21:03.254778 140411597268160 spec.py:346] Evaluating on the validation split. +I0913 00:22:25.655730 140411597268160 spec.py:363] Evaluating on the test split. +I0913 00:22:26.515308 140411597268160 submission_runner.py:516] Time since start: 78569.99s, Step: 112302, {'train/accuracy': 0.8119818239795918, 'train/loss': 4.524528036312181, 'validation/accuracy': 0.60438, 'validation/loss': 4.533105, 'validation/num_examples': 50000, 'test/accuracy': 0.5399, 'test/loss': 2.811767578125, 'test/num_examples': 10000, 'score': 73911.98550271988, 'total_duration': 78569.98849368095, 'accumulated_submission_time': 73911.98550271988, 'accumulated_eval_time': 4463.378042936325, 'accumulated_logging_time': 9.768167972564697} +I0913 00:22:26.916514 140390099318528 logging_writer.py:48] [112302] accumulated_eval_time=4463.38, accumulated_logging_time=9.76817, accumulated_submission_time=73912, global_step=112302, preemption_count=0, score=73912, test/accuracy=0.5399, test/loss=2.81177, test/num_examples=10000, total_duration=78570, train/accuracy=0.811982, train/loss=4.52453, validation/accuracy=0.60438, validation/loss=4.5331, validation/num_examples=50000 +I0913 00:24:38.122423 140390048962304 logging_writer.py:48] [112500] global_step=112500, grad_norm=0.5, loss=0.612583 +I0913 00:24:38.126334 140411597268160 submission.py:307] 112500) loss = 0.613, grad_norm = 0.500 +I0913 00:29:10.678386 140390099318528 logging_writer.py:48] [113000] global_step=113000, grad_norm=0.5, loss=0.545379 +I0913 00:29:10.682664 140411597268160 submission.py:307] 113000) loss = 0.545, grad_norm = 0.500 +I0913 00:35:00.759061 140390048962304 logging_writer.py:48] [113500] global_step=113500, grad_norm=0.5, loss=0.661439 +I0913 00:35:00.762892 140411597268160 submission.py:307] 113500) loss = 0.661, grad_norm = 0.500 +I0913 00:41:18.659321 140390099318528 logging_writer.py:48] [114000] global_step=114000, grad_norm=0.5, loss=0.634262 +I0913 00:41:18.664089 140411597268160 submission.py:307] 114000) loss = 0.634, grad_norm = 0.500 +I0913 00:44:58.789774 140390048962304 logging_writer.py:48] [114500] global_step=114500, grad_norm=0.5, loss=0.598286 +I0913 00:44:58.794031 140411597268160 submission.py:307] 114500) loss = 0.598, grad_norm = 0.500 +I0913 00:52:23.200891 140390099318528 logging_writer.py:48] [115000] global_step=115000, grad_norm=0.5, loss=0.619005 +I0913 00:52:23.205124 140411597268160 submission.py:307] 115000) loss = 0.619, grad_norm = 0.500 +I0913 00:55:42.840517 140390048962304 logging_writer.py:48] [115412] global_step=115412, preemption_count=0, score=75904.2 +I0913 00:55:48.253401 140411597268160 submission_runner.py:857] Final imagenet_resnet score: 75904.17921590805 diff --git a/logs/self_tuning/ademamix_golden/study_0/imagenet_resnet_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_0/imagenet_resnet_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..196b09ed --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/imagenet_resnet_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,39 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/accuracy,test/loss,test/num_examples,total_duration,train/accuracy,train/loss,validation/accuracy,validation/loss,validation/num_examples +202.6214411258697,0.0,121.5451877117157,1,0,121.5451877117157,0.0016,6.91218828125,10000,324.81873393058777,0.0011957908163265,6.911289137236926,0.00114,6.91032625,50000 +329.35181760787964,0.0297186374664306,2115.945389509201,1911,0,2115.945389509201,0.0581,5.43460625,10000,2448.4243199825287,0.0758330676020408,5.683907645089286,0.08448,5.369495,50000 +446.6659274101257,0.0852634906768798,4113.932512521744,4679,0,4113.932512521744,0.1978,4.166992578125,10000,4566.972930669785,0.2529695471938775,4.555435180664063,0.2422,4.4598834375,50000 +551.2657015323639,0.1217114925384521,6106.99627327919,7813,0,6106.99627327919,0.3349,3.22026875,10000,6668.810752868652,0.4213966836734694,4.301740373883929,0.41084,3.6116853125,50000 +681.1285533905029,0.1685569286346435,8100.409832239151,10908,0,8100.409832239151,0.4208,2.76146171875,10000,8796.185854911804,0.5089684311224489,3.690275542590083,0.48904,3.393254375,50000 +782.3462271690369,0.3399162292480469,10093.10680270195,13975,0,10093.10680270195,0.4468,2.5407837890625,10000,10894.439922094343,0.5401785714285714,3.8735943229831,0.49856,3.5946475,50000 +916.060443162918,0.5296454429626465,12085.535720586777,17094,0,12085.535720586777,0.4778,2.466045703125,10000,13025.37265920639,0.5809351084183674,3.806796326929209,0.55234,3.24312875,50000 +1014.7383263111116,0.7073013782501221,14080.839374542236,20017,0,14080.839374542236,0.4969,2.292778125,10000,15124.489433050156,0.5847417091836735,4.2652899294483415,0.57624,3.0346875,50000 +1136.4618639945984,1.0467360019683838,16076.302437067032,23241,0,16076.302437067032,0.5222,2.24196796875,10000,17248.351852178574,0.6491749043367347,3.424844547193877,0.57928,3.2999325,50000 +1251.8468353748322,1.3355841636657717,18069.75966453552,26171,0,18069.75966453552,0.5141,2.249761328125,10000,19362.27117872238,0.6524035395408163,3.653707387496014,0.58686,3.1516565625,50000 +1371.63134431839,1.625967264175415,20062.942791700363,29427,0,20062.942791700363,0.5272,2.18953828125,10000,21481.680223941803,0.6569674744897959,3.933512784996811,0.5849,3.3899871875,50000 +1489.5095262527466,1.98438024520874,22062.18269753456,32376,0,22062.18269753456,0.5364,2.1774421875,10000,23603.81263756752,0.6981425382653061,3.257666451590402,0.60376,3.2073090625,50000 +1604.0221438407898,2.28261399269104,24057.024208545685,35628,0,24057.024208545685,0.5405,2.2027775390625,10000,25719.10164737701,0.7033641581632653,3.4684765874123085,0.59992,3.4122078125,50000 +1723.3271679878235,2.560221672058105,26055.084835529327,38590,0,26055.084835529327,0.5371,2.18135625,10000,27841.924547433853,0.7145448022959183,3.2748126594387754,0.58636,3.774430625,50000 +1835.4817848205569,2.834051609039306,28047.74717950821,41831,0,28047.74717950821,0.5336,2.324109765625,10000,29952.44298005104,0.7114158163265306,3.6007372797751915,0.60598,3.3401575,50000 +1959.2915556430817,3.106719970703125,30044.303684949875,44812,0,30044.303684949875,0.5384,2.2525880859375,10000,32077.29454255104,0.6891940369897959,4.2108745964206,0.60408,3.409203125,50000 +2069.0723991394043,3.439774990081787,32036.41220641136,48029,0,32036.41220641136,0.5376,2.3202591796875,10000,34185.70292234421,0.7480269451530612,3.264349489795918,0.61424,3.2800534375,50000 +2189.3234827518463,3.695246696472168,34034.62011766434,51025,0,34034.62011766434,0.5283,2.405058984375,10000,36309.644293785095,0.7228156887755102,4.107982090541294,0.60628,3.646350625,50000 +2291.069113969803,3.971768617630005,36030.59577226639,54173,0,36030.59577226639,0.5371,2.37350625,10000,38411.74111676216,0.7489038584183674,3.67700600137516,0.61914,3.423630625,50000 +2414.7549934387207,4.314009189605713,38026.40539455414,57207,0,38026.40539455414,0.5381,2.3990828125,10000,40536.13014054298,0.7190489477040817,4.503145178970025,0.60846,3.6531325,50000 +2515.279800891876,4.556050777435303,40019.40243244171,60240,0,40019.40243244171,0.5419,2.3658654296875,10000,42635.11920881271,0.7339365433673469,4.250097780811544,0.60516,3.680009375,50000 +2632.641822338104,4.827040195465088,42017.28895163536,63355,0,42017.28895163536,0.5399,2.441900390625,10000,44756.50366902352,0.7662228954081632,3.4690078423947703,0.60764,3.729314375,50000 +2721.444454908371,5.059232711791992,44011.472423791885,66260,0,44011.472423791885,0.5465,2.424560546875,10000,46843.58191680908,0.7411910076530612,4.541734967912946,0.61386,3.7965825,50000 +2844.452032327652,5.329614877700806,46004.137469530106,69453,0,46004.137469530106,0.5419,2.48736328125,10000,48965.33098864555,0.7394571109693877,4.600998236208546,0.60794,3.766720625,50000 +2965.7500581741333,5.646993637084961,47997.16594219208,72390,0,47997.16594219208,0.5405,2.52101484375,10000,51085.02428627014,0.7686344068877551,4.249235192123725,0.61002,3.8339628125,50000 +3077.8687732219696,5.9023377895355225,49989.45276546478,75606,0,49989.45276546478,0.5424,2.534219921875,10000,53195.83649134636,0.7893813775510204,3.65380859375,0.60568,3.93482125,50000 +3196.0724246501923,6.18151068687439,51982.49979019165,78568,0,51982.49979019165,0.5439,2.54808125,10000,55311.74465513229,0.8084143813775511,3.43436167191486,0.60452,4.179504375,50000 +3306.6501145362854,6.569916725158691,53974.51514530182,81755,0,53974.51514530182,0.5395,2.718340625,10000,57420.78761053085,0.7738360969387755,4.3157329948581,0.60878,3.9008559375,50000 +3430.871250152588,6.843689918518066,55967.39384794235,84769,0,55967.39384794235,0.5363,2.6479666015625,10000,59543.1830201149,0.7844985650510204,4.187828219666773,0.5956,4.42532375,50000 +3533.468069791794,7.181390047073364,57960.61058306694,87863,0,57960.61058306694,0.5439,2.617235546875,10000,61643.68226647377,0.8123604910714286,3.558631741270727,0.60924,4.02082625,50000 +3660.716259479522,7.5574729442596436,59953.10913348198,90929,0,59953.10913348198,0.5419,2.7006576171875,10000,63768.584854602814,0.8195352359693877,3.606179373604911,0.60892,4.214265,50000 +3758.870266914368,7.838247060775757,61947.415028095245,93885,0,61947.415028095245,0.5458,2.68063203125,10000,65866.71648287773,0.7375438456632653,6.002884845344388,0.61222,4.09152,50000 +3873.47101354599,8.128909587860107,63939.98579668999,97041,0,63939.98579668999,0.5379,2.7065611328125,10000,67980.07772994041,0.7970145089285714,4.257398644272162,0.60598,4.1292421875,50000 +3989.0768382549286,8.517204523086548,65939.92808747292,99969,0,65939.92808747292,0.537,2.738551953125,10000,70100.81326365471,0.7916533801020408,4.6550697793765945,0.60732,4.32330125,50000 +4108.448180913925,8.829198837280273,67932.3721754551,103202,0,67932.3721754551,0.5385,2.759041015625,10000,72218.91586017609,0.8127391581632653,4.210806710379464,0.61348,4.36987,50000 +4230.150585651398,9.16890573501587,69925.38329291344,106137,0,69925.38329291344,0.5392,2.8169044921875,10000,74338.68905472755,0.8254145408163265,3.80894812759088,0.60976,4.3623375,50000 +4339.878563404083,9.489476919174194,71917.54116845131,109305,0,71917.54116845131,0.5423,2.781318359375,10000,76446.3592095375,0.8047472895408163,4.569540218431122,0.6078,4.565165,50000 +4463.378042936325,9.768167972564697,73911.98550271988,112302,0,73911.98550271988,0.5399,2.811767578125,10000,78569.98849368095,0.8119818239795918,4.524528036312181,0.60438,4.533105,50000 diff --git a/logs/self_tuning/ademamix_golden/study_0/imagenet_resnet_pytorch/trial_1/events.out.tfevents.1789180376.347fee244df2.38.0.v2 b/logs/self_tuning/ademamix_golden/study_0/imagenet_resnet_pytorch/trial_1/events.out.tfevents.1789180376.347fee244df2.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..a40836394dfc40a91293a96afcec6a38686ee814 GIT binary patch literal 110245 zcmbTl~Nqj&nQ>r_OP)Gjw-za5S_l{_n1nJY5{z>|9L0 zj5(zLH|^8W8~s$xs`~~Nr|C9TYpka1ILFnsx0*`18cNNS$DDlB)3S=1s#;~&skV-` zF0R&EdPm#+9lK-MvRA4R+JDEkRBNuLYVTq@ZPM&HE>8aiy!i8{I|LL&+-=nUZ-9Du z6D!rPUVj5xtF=_CV(&O-iml@$S2sK7|00V2XNV|2?Z(PZe!Nq8`1@GdMyjLGx?ii)4t9G)$5^RPAjz*YHG3^ z@_Y?*a<9KZN(U1UtNxA=gG|(n)#?yIv)!F0 z*?HMIIXl{k)2=kEk2##G0pAbT>-abDWnS`F)%q92KwY)gYE|9rT-~TMUgykh1X^@k z@CGJdJxPoa){hWF()JliXrcCjkGuXolKMIk6I8u!i4lfs`fAmM2qvnhc7H%MbLyDT z_HUHa#D>FE_3w#M&D5HxRdg+W0sXgpbm|oLhT&a)tgQyiQ0I|2T(%77xcz@APy7EO zES~=Va&FAn(+6^j&StFc_xCW~`otMH{+P>;W%!|gn5RoA{2Lygsp)M}BT|Nrlf5LaiHB_5;dto1W{|Bahh zIQx)l^N(U&2etNUb;&r#IrjDrv+W_Vcr2(3%C)M~;cw!-%M)`|L%)fM{GGt32ZUVw z;OW!lZ%BE18#zH1Z1ze$&` zkNK`zWB1xi4lB7PgGP*P`w9=@Yi=`3T0Xm881E~Ys6lR~q(QHMobS^I5Td)We=YNV3e1#7xG+rC(g5rE)u1rypR&4c?D2&t3+yzBM zvN&2ng?E+ds-UpHBS!EQp{Vd~9qtHBr1YePWiuy|XI)b9x5|t#O_f)18mJSky#;SZhP}r^%N3#{w>%uYM#!*lVTrWoO z71^jz-EQFyiWIAQGKC7Q7`aWN&>nr?6BO-i#nEg9wMRwt@xoWP+DtKmuTbg*SDxYS zFOPy^azkC2LX}n=x0NW|Gxauu!k~pXT0w>H&QY~sF8UdW5qyOvDmp* zYP7=rf<&<EZ93xrl>+IPVSK? zlx{u=14Y-v;%K&l`kN-Wf5rKruskV7@D)a=h`+Q~1r)0tC&(1)v|^8oL@}+u&mB;- za1%$f71Ug0PHC(QigVs#1Ygk;6+aJ*-2)06Jtvu>Dy=9CktoJzR@ngxv-aX>wu0KD zR+L*24T@1+#R$G)7%I$P7}f{H=GO~l3JqGJazLUmxt8z>6#ELr(QE~^NBM?@;5wTn2d_uXM?wb;@Y9LGDS67kuQB+Y*49}3W~cY#L)^WM$FXlgSq&8R*c{)oKewK z+58kJ%FWm%Q)to(yEMsMbZ8Y<0E#=#;%K(wm9$5hZ1XMzMdf*71YhBUia%A)&jZEL z){!zrby~4fs#x1{Zy+f0jKtAw1@-Fo*U5MRicy`#2)@D}754{p?gEPO1qm`m4O%fY zPBIt1XVc?Bq48B5%~nuz;dT9ecTm*(BS!EQfv9+{nLis8E6?1PDQeP+Cp#qyFN0uw zUFal;quB~-F1~qHPy@yGBr$@o2u8(mJ<~6suwU{_rqH4l%kw3QIW<<#gSj}kQXH+I z!ue3jD^P^57bEzJP*n7OyJS5m&h;ylDQYnkGbM_5+ja+mqSX*_G+RMkFS;ss>HSaR3!dQDS45)X zVOXVSptx~IOQxtpE0#J*6a#JrJchZb|4D_of_{A3e9HbGDUq_VOA(nsM;i4 z0L4XJaWq>&T`xQe98ExxXe>tX6{)D0Kczi3kjXh?WQqo~V$lwv;uJcLotG{C_=@yfX)*^(Ie-uOwoi^+`A{4 zi-jZm-vY(Qd~q~ekt6;4;#B=FS)dsAS&ZN-l<>_cjV+ncps05+R;Fl5D>f{ZCIr;Tj`G@D<9asItGp1W+uWn<7&*qZMsdNfgiXk52%F-#l?N zTS4tn#`;x*L2-SF7{OPlpd$WTrg2rTm;#hZaG+RMkyQ|#y z!X7omT#Vo=)KSrIvWpQY&VDJ7DVo!Y^s5qucE-D!pa}gfj#f~S*R|sqn2Sf1-u-s~ ze1#?|irgkQ0)=zbA6ZBXT5;%Y7T&9FHJ}P!yFFvY(F!WI)t=q~6dsqv2);rK70wOk z)&s@xrJBjY-xn=uMYlzgxmc00?uagMJrn2A-&uD`MIqrC?;5mqZL$KdGXZ)6k3zS2);ra73)=Mnt@_V17n$@HLb87 zBAE;I;m`2h$xAxoXtsjdX+JkS+z=EE^u-9iq6sQabhy+E6z(s2%M@)Gih&ZvNYx77 zFc+KOilf;IYDKx4tP29g*3V)DU(p;Djdt{W0E(?=M#>brw4#ml9kI41>+mnu+416N zwu0JeGu7>`fI>M*jNmI;qatZV-U3kg_)L>2^k{|sT*+L-|8`4*xv0HT9L-ixbCG-U zLVr-~T`xxP75b>q?y~$nD6WllmnrmV#V~0zS2Ip*1&R(+#L;X8^;go4xPvc2F?^;N z!B-feqW9Z5tw3R=yIiI)pcRFUBy-`lwe?p}%rq89vlY}_Y<&M902GE@#R$H_1Qof{ zmnMPYY~f~^!jM+fu#_k?clhi9#myh$Xtsiyi|-S2u$f0y*z(^2@D-g7ZB^ z7cL8FODnD$N)!V^JT`&i!WD5eTS3jmolpmtjI=@VOlqF|mF!B_M}#fY#*Q$VrWBvYm^rWM&9lDX*owN?=*V$HjZD#=Rv1a&ovf}uvIQtks)(c63hEAL_`rn9 zp!lXKM(`DbP_ZCo9Nytvn)E}aFrgKx>m_rM(8PB;C@fRO(F!V@ubmtOiX9nZ1Ya=> z6?@aWIlx?;+oFD5XyzSg#m5m6#bUcPZ9w4~Dvo9=s68ro#lyOwcymCE;44O<;##fp z_*pey+lDfQDXoZ>DvtJfjGvF{=^&0)P!V=!L=h;q&lMy13TsptDBWxfb1~Y`P^Rce zD^_)s%tdYA2Q^?Wx|)ik6;vF_t`rRlt?pt3Uoim{v9&%{28C6%9x_EITCw7f^e@(E zvlpP4Uq>9Rph9Db)nHIKH5McIipi*GRc$T4$2$MnP?@4Lt=JPPnTrQK*L(oQyf@-# zwu1UAX~W!t6QHQ^S&ZN-rlDf_G`+Jh7q%BB%M@K`#VU7+qTP<+bwJVZnmC%RpkCei z7aI70qW^6%g0Gl?3Zn};k)W{LI#;IXN-MVLN)%%&4`>bwqut_Ywt{LPk!Kp2f@1DL zF@mpfM8%ZkHXA`K)wLJ7dVUZanQ}mz}duB=$ zu}OyxgW~5?akPSpMoq$J!CW+bEk^Ja-l+J{&%q58PDd`w6g_E$dS!{CRE z9L-ixd({2*O}#-;E?$h_D|}Efs!k95++e#+_hbsjx6;N-KdW~2*UJu|=(STE%~nt= zYDAm$$)JeYFGlbcOHt9Z?a_}g7mb&{lqq`Cb1}kNG8cE=JW~RN@mg^-TS3i5p?{sN zpwQYZM(`EBsEF?vXA6pEkBelAKD1(^n?#|Sy6FfgGIPbzYz4JPosaCU0g6`zVgz5Y z92JXPT)TqeOqA*kp_%uk6+VL`is612tUwWXUL4IEO_rb-Wh`Az0@D={3$PLWI zd*<%z>&q1VXvH+?{iqJtPId)_-&S!nTS5J;_c*~K9Tai9#R$G)6)L_qt855!@yb?D zrsz*Asz|?M8Na_~EGU*ah@;sGYDJBxF)SApMRUanz9Ik>joZ~L1jXYfW-^62t!OE| z+k9nf?08TlwH8OS71WB#GPceJ#c?Asg0EPMihvFpcAz+#F-WEuKr6aDkoB_~xuE!X z>$ogr2(1{NBiWcxg-wPynysLAS_`9l_+6O!*7e_0oP_b6O28zf)F@moMMMVS4?8Ts%KRQRI7)~o58%PuvwwwtE#m9-_XtsiS zbwBss)dm!uri&4L#U50|*l6OX<-J>emMKQiiaD(%il~97@ZR7YLvb`)LG85aV;V07 zMJ-b?g0Bce#V9*17nqCwDwR@%W5LcC;OYdz5*FI{%GUP;n{JsyryP)x`+D;s7e1 z-f3Y4b8&lK7uj))rWL``Ps@)rQ=bWQVYE~nt)QaK^V4{@dEzQDg0DD)iownU@jcdi zU608WRS%HCd(@ zLn{o^B#O*;m+`xk@14ZaYz6hYsJv}l2q<#h#R$IQC@OAe)*A|n+po*t6kgrdv_k#1 zMA5OndoHY~^pE0bwt|`qgQ~t8K+*Dt7{OOWq9SYP_g|psku*xC7)vV}9g`@IeKWwn zFAQ#rquB~-F4FHF#CwC|9*7Zq#R*i@zP|YYD#B;T6ys>c`)ZOsYJD#y6_|_YW8!E9 z71mR?_k_8KKO;u)6;Y@-^h56jC|0cSmMO;53gbk{Tr|t8{tFb_*NLOq3hEQubNju- zy9Z~sh!K3nX;eIW@dG~kR!&Z45l z^N~+Lk@3F#En$zcp%n=~C5myIA9MsoP?0#At)S-O!NrArKvBJ1h5rtKuZTrOPOAsU zL6P{VrYvLRzBI3Kt{zic6?CI^=-`%*9JrKbc}GtuUT1QPi@Ug!iMmd5fdj3aWw39XJKQ z-`&7hjNmJ-pyE`d5jK#jBR9(w(`d!odlE(UTAK{vb)h{$9L-ixbI~!&;4sXE>ohTf zuSi0L+rSN{;Td~dtNM>>Tl}oXPmG6o89q(HFJQd2t5OFZ(P&#oKAi0+klt^$T$`U| z`uEXpL&pTXQ{CQJp23HF$0y+Rd|+||An$T&{1PB}jHfS6*&BpyE0rKIad^j8O zMG9FTX#WQw`9I_te8~L1@Z_S8TQ0r;$bxojosEiTB#ZI59HXVj5aKj+UUh&(ca;xk zL%vHP^GE!s0+9WEQdx=YZM_+1?G#$n~|U_<)b8 zDm}#@&VxWVJUfNw<%fnr)9!y)ned3#2AcAa(w+pGS>XsCQhG{9Q+TjpTpf8@Nsj`3 zdFL=5*T`~dTJYq?*v9gte|i?^*$x$+!*eg6wv;FG9|qd@yAv8?e0J%>O42Zt=Ljo_ zdK&1!yd$XVZ#e9p+26CCbY|}e)vg`n8D)GN=+NFRyBYn>+VnYfl4_)xJd1uFXp8F3951=3^&Bi;fbMr{^lL`YZ0m%K31Nm&@(}O+zQR$47T-* zH-de^%~l@C-50V8a9^mR5m^JC5W9GWJf&<;1vQ@h%?|cT&07bug)JtP@d%};dSSRa z-?p~A4=da=Lq43nSN@Rhm4BX^*Mf5rlP%BSo)e|-cuo#~4=DtQVY2DmcEy|iCB`#v zTE5GagdX z=>9r{v@%7%p0dkhWby zJOE;RMLt|XNWAHn$pATXOP;}nD9uMm?nT$f0MYTicu0T*F}}AQxpXo@PKC5T1(0TI z<-^&KpVHSze6@1@0P<&xJcA1f4M51*=l8teLKN}z=^FuZwRuU!sI%V^d{+B0S08+O zO6(hX5T_V5xw0P5ON;tBi?A3K%0o&kM&=c~@R0gMuAKwLX!Un_T1mxd;+eI0Tz!^W z4uxVAQT~~Dmj0<2ZQFmU4HTnlYVt&WF{%|{iN*@vE%3SYp4H_!!b+lwQ8{Zr)Wyx6 z?EuB-N?mzI8H-U^W|lV;qa{t`S#&Y-Sm%T5(coI2*RUS_TFJ+kP>imfS&KpCPCp$A z#mK}^9;9e7q7Ls;NTYZtMuR%YQ%f&K-9|sbVzhnp@6}L@OuNbR6e>ocJC9;j-ra2P zV%Qg6^_53*_k~Uy@buZ66zD-Q(iKXrF!j)uU`I)@A7}VD%mTS zxJ(-edu5|K^5N{g@{e?{+`CotGMtk?jpZ5KbJ8;e&&keCA58#aYg{x{fc#{9g?y01 zFnpP9@6_T6K(?F7hqED6F^cp#J02h-`pGl6knF<qfrZ$_Chgo7%5NW7o!SZhtb$9*Zdr;q=n<;Il@Y!iqW&WLr~W$ zV2K+Pqt;X88D%U+c|U7LLoo_+kY~}wsOWwiuE!ydX10R$c-~n)zJy}rJv|kJ9%!j( zLoqV-lm{tVjHtsqw)b5XC`SDk$x}-&M&)j+;!TLh=d0g`Vie~m&r_%vW%v1p#b`i- z$Ms-esJd1j$=w%Pf5OwZ_;&IsC`Ou_7C-5S>l zSLcTp$#@yQc|ks$y;o9wVp_;9JSQz~$TPU-MCm)8lgZ{g3;=Tc;tn?f62@Cos9)?#VN_kW^*-m8b7n5vc%a*f1|yfE-|a`6P2x zZG`k#av7^?|5ozh3PL&_cNhSWBS!KJE+kYPAr%)~91M`u{G*jki{Ntz5oeie7sxxv>HMh z#qQb;kkrZY;cN(XSM^?pYApd0G((=jg-Cxu>~Ub#8UWd`H21Io@nw7-x4C0ggj8@< z$2T{tu8|LCL%vC8ByncZMu0rsB+uYNG@Ij$L_}oy!iA`5vrbhz3QKErNyX^G!1DNm zT;a#(;di8M^yNXEVl>ad5zmXs@iw=i7|m`k4=Jq}xla9#rQv;*C00<3&UcZgl~jzh z^v2?G)y$5p4aKN)A9>P06{EuYyYrzK1zO4z`NgQ7tE7*1^IdNNFNv4K`K0z@$G+CZS7bA0>UwHA^xTbkMSdT_CyMsd!n zxUhX2?LGj-==p4UkfOziI=ogHT6Lfp<+{sLOD{%U2IyijicM+#4T_P^VtJlI#mH!z zKCb$2X{9=G;-;xh!L#SeO&ph-rK+^BaGq{jpeGt+|dE7gI^yya3N`NHU(VzKp zO#O^A67hU?KY)0d%ZIZe)Mt#MYWd;2v}VKP8C*z{fe7(Ek=FWvqX)`|vmsP5dQsUw0cPaqNO=Yq@_qzD_P<-y0WL&egRa%dd&k|+IBG4f4*dI^eA<>$LYkq@`^!jWC`Kwb(Mn+KE8xv zl-OBvWnZnKZ41TdPPRNq(PBg$-mBgnoS_&g=gCt`FGg)P_rYRx>RYxO6r=ce@;rr# zk&o(8ToLKo3r4}duDyPrQqXwC`QS@s!=3@h9*NIsmsS5j9m@AZD`;GApHiWu*vwE*$`?*8g;U63^UU9fINc>F>*mjvxf8b0i@-saGY>H+Tn@OUKs#6fAkZ z3`>_WchF+k(q<~Wi8dmJ7|*s)Ti4B&0V-gzOC&VKp4Je7XNK7%>N$B*^586t3JI+Bq>_? zr~`cSQD=Xzzp+XO6A!EU-z8#8?|hFd*kR|ZS~KWQ-@k!<#u{!@o&JajRH*c2Y_5#! zKTXSKJ?s+oUJ#kwT_V5_C&R0{{Yv;tz$>4KDO>FulCj7WcF&Kt=Dxy?ljYCYJ)K*m zA?bHJS5H{$dXB_+_U=ht)Y20x4TAHc?oNbo&r7sFo|mREk#)e*^2gpd!IB!luyh+N zu`G10s}C0Aied8cYzuWIxI1ul1F)3WAVRp7&;YdD9@)nTEXn)2)Hf@>5?l&oSnAqJ zu4?M#r=DK z{ATHu5yW`5<&*Rd@}5BxuYqOd1R{iMu?|Lyv-;6oupE1{zfrg1H5MAeuuM@-L`!t7 zXU$<=8h<9nvn|xTWOx0&8Z4`S6Cqqn&k(finD#yZE>IOmT6GmH$Nz7UU7%cS9cH(+ zojTRs#dhkv-fAl4YA7{R-gCSbs=VH62SUSko=7Bdnp3ZZ{&@Lb9egt#n$rk7BBiwE zGN<$Oo~~s`4BXx>AQ#|MVk|KfRAf5>H^K_d^izXdUNV^ zY#}zMDi@qmp*h_?N(3s@oT93C!1e#9_a#l(CAOU=GP%3N6v-~pFgm<3G^e=pL`>P5 z(?PWz7O;E9tIu;0cATJ4#_oB=Npe+due|m*to6{^#CZ1ZNxcO(?CX=Qa9)%e6CvF5 z(mE8ER?lrNGr$rNu`XS(_=hnpMN_Wg^y~=xg1=$1^E5G@ZK0Zz|BXkwV9|*uLbw*q zFtmgZ)ztw@=a?V5-HWekKf@W8v30MZB_RGF{$`ZNC1O0=LN%wO=avS6<#h@X!nG8I zqosBycU)sp<=)#1miLDl7L$Dv%g1kB_QM)GsZNYn&{Dx^Kc#4BRf7R$V?*YpPG~3_Et5aIlY`qMDm+c?$L^P z?rR*@iiB53?|DR!u%f8uWba)Kf%T_741nfz-G>M%V{@|4ef|lWQ``z7jBZY*?H1s| zGz|S52@A7uEfG*cbBan>j!Cz!tm_BO>EtFNNzvv+9pJgSVXvV%Wrq;4r8lP}jkVaE zs$SG~fac^9Mg%IYFTrQ< zyv)k39SoL)Esl2uOTsyZrLyxev@{rZ5`PtO=5As<+d{p&^>&oSKCp~BM1*iHKIhO< zq3N~VTl|enU~)>1@mAJs`r~1&$dv_Y1)H3I$*h5sfzp{xRx35 zXsI#3;S;c!HeI<|upGF^uvE*H+^1<^uZ2&HoT^KVXIrQ>_V%!PXR!2bPlRwS!!Dxb z{QLL)!D3QvR;*wNxyrEY_dkJaY_QTHe75bzdc=6Pg}P4@rL(j>SX$^1AzVwRt7!SD zePJ$G?B;gS>Q%hP08!i?oEtmTc~-_KRU7(SknB65U!=kb+p{Q za%u-$pstu+wh=7nZj{)Z-c)FXGZbEV^G9e-n!Si5PII!p=YW@Q`{pg)LUXbhK%|t` zoZQW7<3a5|u)!Fblj<-cuB7G^7iED*w(r3o{ABSoYa;5On$yuW0ol-;zE2_|`OT?X zNCjjLTB)-FR#XFfB1l+KRCB8HQ_`F=8YJsMb2>7c2q|N8dY_I!aXl4 zxA44l&b@XYES}4&+z>29cNmsr?+CPb7kJ}GSs$+>#5AVYo6waQub`z_BeynSN$N|4a4iQOqs1YvqAOT# zq|a&Ar+AGeJ!4o_hDciArXf?#!Wt`jLX2lyK1%;;Iv<_37%W<^h!CzN@EKa_4QiVV z7bua z-a;&I8Jh~tsZ$0KS5k9Y^Vu4YY|yxq8F0^OY&H?~Pt9qwxvD=jr-pe%B)>TYeW`@! zeogzsqw7W-*^^94)fYlh|WfE{RgzoPAAuq>NRjAvV@E5Wexk2ZkC+>;35S|-0n zOR9}2e$MHscV6?p#aFek0){2-T^L$oPdw`kmc$jrc(#R_7whX&euKq&JrTmS3@AWL zvf7yloR_Fhiv>&YM~0TIzm5OOH+|JHgVvexPQ* z;x*>;onZ;8BC$NFpLz@|2F;1_YzuYI>4WvXDqvAFAVRnnrSE7t{WE4eT%f8=ZoN#f z6#Xo*IeEnx;|vX2)NU9wr^YjhBu;bMzEJYyxi|G!PKD+aGnYsytvM+N*2jaIxMvJL z8!*?4h%2!<^&O1GqQ|ne<)Jy5EFq%)sX6&69WaFEbkLuOpd zfD-OGh1y4BQg6ozzo9vOJxnAi+MK8Z{4}KcXlPD#PZF`EH>V*+XR$f`YFaHCn$xYb zM4&>=NnhzVE{w~1TkuMhoJeGHcZv5u@f0Q$X{JDPGEOF9%GR9T^?kP*c26bOej9}y z=hH97?zuiu@^|Ns z_dgB#7hlz~{1}$x;QeShb>WjKSdM=u#@odWn>Aj8iL7O6BUOs0MAzVwUGFlqn{T&XL zn$eaCg5}a0hNW$rV4Rm58jD`fu;^#6K})&M88KkFqD72X(DL!*q$@Blg$;=it|eLpEuI>U)4;M-(|)>O ziP*%jO#2*&mVS-{gTRtmpBT@!P;2bS=_O~uGOsxi!nH)Gp=H~R?#T= zM3Ates1`W%yX5}alJ4{FLJLg3M1+*F1D7B8le zVaEyG%Gf=Z-rtUEeQd1xO4vQG8W7{zyC-#3>zdHp2zJk1U5F6wc?ng=^YW_Y>5E_) zzPE0t0mWCfpdAd0M$R_0bgP((->vI$gcz@&rG4tBRIuznO@weQK^kZYw9LoPY&E-B z$3w9A?`Bv+j3n>Y^)mT@kC5JcgBY)%WlWIy0kAZ=ON4MO{+ehRnp1EY=4GFQ@hidd zGmK&JSJ{d4qPo^F8`jtXS7JQdLfzNA?2xnvEFTvVAzX`h4YY(mxQn0JYLow@zD4mG zdw+moX);*S0{f0!I|VEjUx@K+3w6(FRjpgtHkSS&Lbw)ZEwq?q9p4J`@>X-eT)~nP z!LTH_NZR?#M%t~xQdEx^&$dv_X<@qtA7Ng$H6uc}mT9%oV)1BjB3z)Je?PKcu*4lH zu{q7iH^tSKo$2ie&8bDjqw*w9bMoIj9Q#Sp#iVM`oHnTwDWx^1$Oe+nY)`&`S{)*i-<+l{mfSxZo2lgwD@sj| z2ohEl)tts`m)t*_ULoi_G^b!=BBYGXY2w#eOQ1Pz=}d&t%}H;qXBJy`Ytn^TqNRi8m~>SazODcYQ<12oRbybaCCWe5>ldUNucc^{ioaDRsb(42-@ z5rGOdr-)1)T(n)!o8YI1+t?79++8A0vP(SC9(4_xlZ_n_Q?}+bDDXsA*gc!48$S|u zoQz1u?in;P2=|%IS1)S9T0i@U7|-54srxi5@_TNC-7_ke2;rU=>$ah zvve$ruWC1<7?w|i)}v*z#TjL=v>!!`SI{!})8`{#(VR$xa4m!Cqh(miI0vwNTsVD` zU`dE(Sh8a_p{2_aWm~WWuOP;=EmWg@v1$x{hwIHcB7|${sg0K7zbXZSCGY&ZD}p5| zmSNGDBDuFQC#ycbyZ!DOF_7s; zQq?d3=jCjDn^rI{x{ZnP3R)bqkCg+<_|`-S*V4KPTAZGL8w2yAZkFjGSVAr_EYqh; zzPxiT+;=HhF7_wJvn^Dktv=i8HdxYz5g}ZQb~ChC4?TGeE>PcU4LvPbUS27&IgM>B zX-1G$-@!L?pjCmEWd<%)p8kb6`ce^&^6W6-70t zj&WKD%-mIbJ~XEzgNTqaHmB#t*UYXj-|s!@|@YPXv_EoSrL5 zK8&*IS6~4&CofwfNzvv+9iX>PVH0Rh{xgW!(woz)IWMp|X|5kW7@E^fXChFc=2YLd z1J3F4y#?iAm$>akWO8>2dmY>*noiEKg68yeF%eU?<`iW7ycO)8>+g7`2s=(-5@Yvl z(nWHGUiUb{8`k>Rhs1dH?n(XK`OfyubT}{3IYbEeyl6JZ^K!=J@-whl_IB?-sQ9Yp zdxK#aws|AIZiB`@Uj~*MgNX5L3)KQ6E?vjxY^jVULbw*?mS`Dz!{jkoPHvjBNU#*$ zVptjsUX7N4t#a_Y%6Xy0c(#RVPWtWv2{13VhlmiarKlBJ7B9PM0+yriqwWfp{4|C| zeajNG^zBlk4$O<*4`Mvq@=p5BW6Ccz4Y1s8VE$ego*Y9TCE{q#K~+%|knHxIjhCU3pEg1UxRW zIaQovgfrB=_wP&4oT_*eNu1{79lsp6`lHzm3ZOapFC$V)YfityB%j%eDV&3!c5lC$ zh%2c%O)-+ZTi5<|eIsa2t2Po*|J0luH|g0!b1J``h~zh?P3e+*PLE$ao)0Uk#vUR_ zSW#4Sve(tbQ@`Zx1Yc-Q7Y-63Wo%B1+*~R_b4rOM!szC-JzCOt%3WA=8y2R1G!alj zbK0^^^0CB&%PLKV=HwYiBq`dQr~{nXe_|{&rX}oWdNo6hd=)mO=z7 z)SP_M`{SJ6F|rvAJKpuXL?(BaP;ZO7#QbL}_=JVDEFz|C&FSZkr>3xbZjLN?CG0p! zPZ+!B)^N$^HWucp-i5VpbdDI$-aX$-uWC-$voFDUX?ckV;hvWy$$2^HI|o0BoD~|? zVo32-E%rIXGJmY(Z)jJGFBM-pg}OpFk5_#S^Rjdf5yG`ZwnvLr^Y{1+B%QAd zG6YN58-~SJPtwjCYHgeV7V`?;^6_j7H7|W!e>4G0ld41r*AmtNEiGd9;p?UR^aeGD z7O%13cMQu$BZ=jx^3im#d~znnvn|wjQEzSZ`2?0n^NA3yCAcG6`gV-30c&jk*0>dd zC7_UDS);rTJI1sP8oFS~+e3_JTd3xwz4jJf!pHO;YQ4ds|Io>oKTA%WnFa^EQMc?`6`lt?0_v=(@`dqX@Zhjc?N zxPR6>nusf@1={6H-V^<;ZCDLjpiTl2^-nGE^Jn8kXn}TDiAa76Y!xbLf$sw~Cc=t( zaFYlURut6&n=F>xKf7XNw-j1nzk5VT8C&3_GvyjW3+(xb2%}qI+5ky+GxPc}4;H3% z4iQj73yfM>5tq2pJFA`00&`yzNs6{W>Hq@_C+vn6X!4PWExiS{8D0*5*HORr)OFAT zRlgH~3bnxgEhbtSA!5qb0^6$ibb~9w z;-Q7@hZSGdK7D8Go}nSD@Fl%susVM4$$Ts^UcudS>!-zI;JnS8Td3v~UpuBRSeCjFAzaIh9%w1-aSWdp zajmQUb-|Lgv;zGtgu+nCJ*S+4;S0d>svj|)ZK1Ae&SMA7152Z!L;=J6lNW)LNpY|ihvn|x;oOCA)O9so&HAD#4GOQ0;T0eU3 z3zn!LI~T!nX%)j_zik;>z7|I8f_d4uix|(gQ1_cd9?h)}7S)492-niLA6jk>>$(;! zHurTe36|)!49mW!5=*9@V>PgBc}k2|&{BBvIQ|ww^lKu7Yw2W;7Ui+-2jK#hY!dQD zu-yK?rG)0>s@552Xt1uoE;OfAJ%}VubK2sz47d7nuau*pIrTFqQc7!1dL1P1JWkB4 z5C+X@$`B&1q~>&MqT~r+b0b?CKyzAUMMV8mb8?POPJ-ricp?$WZ%&81OMd($@Z}Ty zrgj56B1l+KRC8LGUk^{cLvkxMXikeAiI6fjrymW%H$iiH=Sqap&1u(1$ya?ov}yYs z7Uq}vL_i76Y2V$dxWv~T>gNf~Y1UFANzvv+9iV%=xHxD|F)NAK(wmdPT@`Fj?T=a3WbGy6B@s>AMix+XDR!QE4*U2r`(FE6!;5bk-=x4@-kY3bYq zt_1mmGUf@E(9I0X?5W;p`7m!4-uG@fh8WMbQ1=7uJMT~iOOwe&2-nhNAXW9 zq8pAZUSs|{85UiCN1T`HdQSK&W9u#xjK55)SOY+c} zZi40KZidAqW+__q4M$vrdAU567_XqkSjTiCShh|fLb#Tn!_eaP!gCGG%lrfRDT3ww zK8B^qZb_qE;bz+iEFDe}I5vxsv(46Y;Cz3eLsrsl{xYZj4oEs0#$@~bBQd)ERvbYg0n|;nR452yQ zjUwVoYEGB5B~P6{w6x=TXiodi6H))voEis4|Agk`afyiJHz(@T?uDO}@s|%)T_=Ks z6-70t6))@HsqgQb=Jfg{k)&vIq7HDVcU^yIPWtbN*wUL*gLc)hIi+||)@6Of}EwW(u^x960XYZcWRn78LP8~QeA$y4s?s>@>h3CbzZS99( zX<%-=N3diZVOXBKODu!dWv&Cu%VETLwuQP9c=QU$0?UT6L|rI>dOkg<4}H9U?z~CEJJy;aU>Lp~ZbzrBtvCS!}mbutdc$ zEYoMX;=GK^oYDX+E~|<0Yzy`Ivqom#_?Jdf5D~()L`^`;q~7f_z>>T=;j&;kaGqfq zRnHSG<$kCS2TPaD#CW!adcAlr?1|qFvDig~a4iRH&=R@J^afm@de-f%ZdJU-zQ&c< zoPuk1Le=vi|KZS_Ry8A%IL+x&y+GXRvwDp;g64EXmq;nCIjLTf+;i&L{YW%4r;5fz zTuIF-|AyqA(~Ysew?T92(}jror{?s+KgkW6lVdL;lHZ*26D04}xo9?2g%x#m01+gt zD5^PaTqU{Z)Njq&ebAhm4I@Iz*qp|NoL&UY$!QD`MmMJ!t0Z5>eY+#F9xTiyHbg)P z&8exijurq#+QgGTXPzGWn*pFJyUmN^%HiS zkPD36(^1_E_nD;TzfQnfUmZ@2XYZcWRc)=M)_yoIGa`u)?s*BBjOWG4s`p#4JQ=RB zOt1uAW>|t}E=0?Lx4!snD%0`Acm*w!te@c%7duZQLb#T|DQMX?(b@tmcj{_A5iGvf z7?wCM$vclNL#M8Rc^T7;7|*s)?+Tq*Xm1VkGD43C;aYsBq2+y1F8=aTa@{1;F~wK4 zq8kj0Su@EM`trBw_&ld3Er{_7S~@+@!Ec%@YfFT1End^nvPG*3elD<)@oZPYlAp@3 zjGZJ|V;hf6$c1?s+Laj3wotE^69&pdVP0DIB|^9s#~EnZysp(5SYyXF8@&`P+35^R zn?lL`GyB&Y@CgfzcN62;7OFWVgg?b!Vpcv(gm5jBXQIW^#4>IHpgGmBB$7DIDMd;0OUzd9W_N++^n5sxQd)Dm^hWY&cb%cl8$xqx zIF^VjsX6ucliUppcDs`R&1sS?5%o{aY5j(Uy`VX{&LATB&FOQD=;@RpTo9r3o)L(ds0_|aWA(2geyV6-9!lYyo_?h^OAC}`AE1D zcrMQg7A#3w3`-j&$!CM+POpdG7*`G?#cn`qg}S%VwRxf|SfZK|AzX`@Gg>|#c%B88o<&)0 z#};4JB6AoP)uhE}Nvt!VKD3Q#g$UtVj9kz%`pOCyu-{w0xf5c`3jbpaAG{$LcR8*o~>yNmKn#15U!=J2U^DbQXL5wsMj{b^u`sh zu^$B`HmA0&`r>NSfApmhG$#!QB8k(SIxE}YR{yqHqY2QQE;th@r8TEPYbE!bUar_& z9h%c^FCwm_=2ZL0Y&^1dg-NfWIW=BFMEz59su(|ODKsZreP*GB&4k{k4BUbK10%2&0>mWmCzQ*Bj1>cYuW% zx{nAbp*bx{H^e2rV8tPKXih^Th$Kau6Lo-k<39#NbMiY$#FpNi`mJh?&1q_cc@t<( zZ_g2d3N@#Op4)Iv0|J*+hh5_4MIw{CODyojQ)qqpMl>|1Yso}R*_u=9+FgFYl_2_< zWe;J;33$)gJ&h0A;64*Qpz|PD>(ye2@$B7`x)MD0eTbj`dYMRsaLn6_`xr*+^l~+ zTd;ik%CKC1G94`&Ouvo*i(?C7JljHD)vE64gr7uKYfFT1EuR*kz~SoW(DAzVwI4_dxP%xnx7sPDJybeK@Q z#?t+iOZtTPiMVd4T5T(F%PPoh?R3Tp zR+M{%y7E~RRut8ol=K@QuxB%q@6eoHsSzP%Y)+Hv*Y$vIdj|eECIc@r3ic7qo+JO{kPFhWfBt@GOb%0-9b+``AX=f`Uw)EztT}Kz2)5vFz zzR;XJ+Y*5aH7BzdyKzn}W-iNzU7}-0B9psI%v^$}@Ym$1bI_ceyAv^GYfiV!SDl3` z!L*lMMhH93r4@|b^Q*BHuJv=R2ls`wUgH}vp1pfgSAv7~=N7_wnW5ZOeh}RA@=S7G znplM4eVQJl=5G}&(W@DjHwBZ?(rM(VP_UetNQ_s|a{HIV2(Y+LCqlTEtfe@|$3}*1 zfqD77Cr@Ky@s%KA9m5jbK+B1sZwlpN4sP7*C96Tc|5^lN-8M!7}O^5yG{k`l4l?p_L(6 zPFU>QE?9!LFf9I4+;EL$9oqUFEPX~2Se1l67FIJAnw{T9WfxuoKBwoG9Q}LjkiQ3zd2Fw){Q#-cp>oy*R2~g{;?XG(}x;FlA_ItI>7i#U8_TLs#TweExkFJU2TWW$@7VQ z6f`HTW<;Pu&B-L-5YFk*mrL+-BN=UoOx`ZxvwsEd5@`b)>p*jA+>VGTTXT9Aun50p z-8?R4mYi~7`ay68iDd)RNu4=yL{93?TpZz3vSA23NHoIu*YQmYn?zi*fiEysF)8ck2dNKDQ;tvn^C}%804D z3M|#T5FuR4iIr&aIHf-hEK99ahfgZLs%0EvSgu^OL`!DULszhrwDXn|#A)VddfCE*0a()`UVw3w)cdcnNp#}VV%7HW-saPHX+=0z!q z2;o{n1JE)r!C)I)pltMA$4)L@V^>a<*ql0Fmb_bcIC*0*G^YqtB8k(SV!y4xt-gxZ z83Sley}J`Br8TF1y4rZE7aV$`0nMpFer8g&~37xSy)wEdl4VqI$e$+v-PII!i3x#3CU=*Zx(0WNCgy`? zLUS6kg@`Fza~hkTO2D>}i2|G^IX~ypPY0*r4NgJO$J`{FOr$fYe_U=ht2|k6@ zNrUr}dx8kzo|kQs^RgtZojq8dt-rNTupBtcu*^}D{0;4ucMYFUb2Nk)&$dukf)fS~ z_+&r3{X_`Y60{cQ_;l`af3Ua*ZmVEhd?g5pV_4L7JK(&uOG{h?^WwXm7|*s)ZKIiE z;t80SBm0ODu4U~ywAkKs_yiWM8yhDGmcT@YrEe!mJKvRQg7*S-?i1tL7HW;HKH}*D zmQzoO5U#~P5G_wa_4hvUELKQnv4W3YJT6r&gI#yvB;IF)W*EkH$4N)cOv#K%1|`c(#Rly)?PJ)fg=A z{tzKti}wb!#F&QP0E=dixE_KfKZRkra7FT$MoL=#DtNsNwjjo{Ez~c{SJN7Z&kWl# zf(YSSJT{_b)APxT;9|S$SZfEta_?q|EihM0@;669z{Egkfxa~Nci`llA?m%Td} zTA)=15y@|XXEPKw!je`EBAsH znEI9oqg$X`Kgq-FUU_uMhK2d-6A@5C3%qw-@^F`Zhdz#g7P#UEk)&t~qz>>(_x;DA z1#U0jM*hkvy#?k@?1?Q<{psiX&;mEB5rGP|z)w$NaZZo5ZuJ&+ynq@+CU=)OEZHSi ztNUJt7Pz_|5mUAnSiNbN=CFJ2>bgEk*m1INGj>l^&k=Yf*u2VlEbN}+`V-^XyC-#3 zdr;#uzAH3+C=tRvFU~=@dp6CQdlxKb58i4`ExxLy-(^^Ccnm{JZIx{XV7d347_Xq^ zLahICnB#&1B7|$1xd|;rnlBcC<+t~Tp@Jpp0mD-Jx8$ledCMM0u*_UejAvV@EA*aj zZs+OwM^TLmaze2I)PI4|p_cgMe~-)j-$*%oSzjmgr$ z`)8{f5+PiR%@(xu{By7;SZXe|FA^+~PZ^dsI)l-&MXB*rn3v-#iScX;H7`}2g6hJ& zBy1o;xEAYRwA^wF*$I~3I@T7`ieE2bFBq0-(E!MjNYr+M}{aTy(f+g?&7TE=g`8@OgH?NiT~?DXlrBU66cFaGLhTP0*Z%JtyKyYEC|Jes~d@S?y?RXif=lh^T*R zPNAOtUqW-r_((+Zo0Cd6$!E6u>cozL71j1T5hScAsySV(Df!i2mkx_QL33JLPFKDZ z%GjJdz7-sY=JY^?2&0>mkH$b;m~+cVOofHHTaySVp*h7@lRT(q_UQ}5pgGO1LnJBM zoTvlT49SUs<}|kv5nFn5YIDIHo0G+rb>`5VCbc926>3iJZ(hblJF;?*2e9J}HzYE- zyM&u$muS>Cq7yWyz8#2|vNfk_lhh>da5cJpR@iZZUom#i-`%?7K2z^#G(OL1sxC2} zy?au>isSJ8fjOL)5ECMVdtL@@!$sO}T{Zml?#k`)zXeOcTZZNM`k`p)nC?&>UbpYV zi1BO-btNeDpMk$#9C?HY;ad7{N6Y9F!T8yrR&{DQ+ZA8cd_FKNyIx2>f3_(2C4T#T zVN+r}+d{RCKf68f)9!0@i4d-(X9!x}o^wusEA;ElQ^Ewxr%w!v%06qH7mf3AyJ22_ z?Ip&uE!4d9UoyZQ*4VS7LBVa->nFqVs(Kf+OkH#zf9+f8DluL` zOQKKI6tMJ4BSN?q!(C{Z7JbbJE>J5v*sT;S4}X`~oEq2fFH!wkbq1Q#svblVr#X4o zmpn}8yTNgDXijhY6Dg%Nr}7UZ_nh`^%20;p$B_5kbO=qMFmdZw3goPZ{|Xn$ssoBBYGX zDW~U^H_)6mxDjD=bMn0>dED&cs!D@lVXj<21eDO6k_Sm1AT-ISYAiG-i={-8qRojq zz!TMN8$fgFyOM}4y*b$*8iLI!%J`-OG^c^tw{X(rd(46!`iI}oAr)}p=@?iJesvmwt*l|*QD$<|XvN$5SA292gnFm}6R(BxA zE4X`hGFFd+^D?wM5yCq!O07e2_qcfr!sg9zbTn(ao*Qk%f9FfYI2PE8an(f$le&LK$)v|VO$4=nSN ziScX;)o4Elc=rU$)pR0+Yti0=mO%%c@xJExo>`HCB_e=f(W@-^oYUL2XZL_5-GUg; zwouLKx=J?_uq+!vgm5jj_oC%&m0O)*URH!TRGU$}#zF%bmOX~Oag7x&X;29)10#v? zYzuYIslI-VT(G>2Awswo%`mh?W)D>d%QyXw7J?-xh+#>+Bx$s!Uwp#Bvd)wk&$dwW zvS-51MPOOjg9zbT)b^p}M*hQYaDj>lC?6tNc5kj&(mki#^Z__S<11vFKyz{nCXzVK zDYCremzXs|qtu`|1@9(ON^4Ht=hel7`n@2j6Evq!2Z^|nnp3ZkHFy!K)3x1cXihIq z5>fxuob-NITmj8#dMpviZ%&^rB~SI-I(tkHVxTz;bNa+r8$Scn^$(HB z+a-JgCA&nT_IVd*PA4l>kna*@YfkgOooNcYr(epHE5eTBAI#W2bHAD4OIq()`3R|n2;o{j9zaWQTX%dH)gw&(vtY>xXIREGm3#{=?cx1N zu*Nix664txYF;$V4&t|F;!hJHTuc5zv?z@ZTLtr?dnD0*X7TIg#$kpf-CJ^ZM$33j z2e7m|LyTuzsCk)f9N8S^B`|>q;aYMJp~b$(7W|%Ip9(Dw3YIfROKeULzexV(DClIS z0?lboRU(PgoOUHl#jRf7E9*Hlr~0*sl+v10tFMxKPVTkhdq8vAs7=I`)SN;N`QVY& z+j+1En$zJHMASbuC-eNO$DldI>l2au=0v?)_pC=uCs6_UTnY>-XXZvB?CHl_UX$;NDVKxy{w&v75{f#&5o>58HPYF9t!ZF6~ImT0R&uK-6 zQzK!mTcr`>*}EroC3taRTqW2&gC7wgyz`=zEjce9)%Pp}OU%ltb!Qb{38GFiEWx^x z&xZEcuwNT2<{OFeYzy6-+&buiB_o6g;aV~yaE>DytS|?Q>FfLR1j~Ul3`?i^rr0)W zdp52ZaqH5qwC>U;{}WFMTRA>x8$Ca z^4pdJVP49gBgV5W)ayn4XLU1}mv@(m5UwRL5-lTUG{Wx*c0XzzD_D+SDX}?erbzB^ zn(O|l2+hednn>a_r+wF_<5oYfMZYv`P6itl1 zH|(y;fk`dITO8Yo2H4&xKrZ?JGHAxH7HbtAXXhG>sVS3wTX(N*wsisXOqc`m`TE-yl zh?JBi*|LX9Qz04&F}>ql*L(SW?{lBWqd(yHz0T*p@8@@ap7WgNK9Q%uoE%=0kY<|G z#jY`Z!JKMKNSN51GsDY+mWqoTKRHdC)8=?PG^gOD?JU5YlvGHdT;`OLQGs*%w6D%Y=n@qjNv2enaQ6+t zA+%VmZ3*U7u0dj&XHLe|1Mrs=(@}k+g^pAGPsb6Ru%Nb#dDrKcj9&PDv;`ALJ->U3 zzIP7${q%hp7d2B7A~h}w;mo)k9_57peqZ>YqSyL{U9IE^hebb>`3}9JaJ&h8ZpS2( zdOnNDHab|{$20FQW|I&}mK&is$4NR~hOk3_%8s%TSPGIjEZveA+nD1scP6knmyvot zi)e?QJ8`Hx%u7x+36W$8zJ@GsKK|z;uw*}qc_y%=rEpjhtjD1_nK_%{Klvv-BlUb1 z(Y#oWtHO7^m1L0+NtVlD42#!foR^qsZTz_MSDgh;ZSzJV+YV$Ym~4eC;OVVuAcktfrfu0Cb%qK6fXwgq#_k0nVG=5)SW z6t;T(ecfk(Iqgd#DNUJ^(;eo71?}$N)q^?3r;|8Y=5)7o7hYN4l-fsNPCN2R)Hj*a z-%~PsgE@u0CXv$Sbl^7ggt)?qS`S!Js-+}ISWzN#aydK{0T10XdjaNTQ%ORaX-=~| z2bY66d4C~cVsl#a#Q_&)LYsqaU}65ESi~+&d*>DFtnD^Au5ey$zv2If87;l&j!Biy zdR|~o@03ZByyhgjz?1b_{lT1Ob|A4$n^S4&8Z@Wr7YeU{IknLsfpVEsNlGow>1gdF zJiYQtZ;~n1CESHDp5Hx7*=K{MDhM$&S>{bYg8xO$+NQHzV5zL&u+%Rcg)E(` z>bk=kv-cqNd=`<>YN&jf11!T2kq}81mvCfxZfM~H^P=b7|GL2P=O+%!#JYjVqR?U# z-lLs-n$*k5l6LxRC9sV9jf6tu%+`g_sqf&^5jBAX2`frufu;t`)9x;p%g2BP z4x36snrVS8dsxo|3+!k{!o(I>7sK53s-8G#A}q|rB_u$`{WFzV=5XZ4J09zT1=`w> zBzY}Rbb+(AVh@7_zF0$Io3_Ax-5k*Z-`rhg2Nw9*i3G}Jfvq|yPBu|hX``a#WV>SB z3P-0s!&YlSml*3tGNrnN`(CC?xD`g1fCY}z;HKRy~)VL^Yh{Wz`QE$5eSf1EFa1mIN>p3jn z-5iQNGvI9|o`Iz5LhAV}BHP%t#3~tB3U-kYNtRXjkR?L1Gg{z*U2(|*%Y&V*#rHL5 z{=__0U2Lwe3iFb2fYkF@ME5oppC7*mSaf|zh$M?m6tZ-=n6Vn>#me=kVU7)J?AC4$ zi`~I~I4?yOBXxk~&3;nPXAzl`&7GOMfkmm5gh;Y1jYgJ7&t~KM1Je7g`cYuHx{t#$ zr8o1f>3UMzufP)jJ*nrjh`wp;EB5RIEFbhqh$M^oePqck9c&6s-E+d~_X5k#gRNyf zA)auF`OUu71=Ukv8wE2-l7ww6s`AD?-Me<|b+8T7`6Q(&+gPxTv5m#PzrF?AC|*M1 zWZA}cOXgpH@x9&dfNjjTBT?UE8%{mvEe6}DUPmIOZ6kdvV;e?eOe$#B=!8QVJl9=Y%#>_=~-C-v%O!6xgI?maDI*#bq zy3XgAonV*iIxSf1BeF<6zk7g7U;?%IK zc^~1h^jM^gyIMr=tM7qj-6T@aXA$jc<#n&|RDvTjNQfkh$pf6@S&QdS0haYGb9W0Y zdyjKi(#MQJmK9FD&;sAKCG~t3(XQrye|7}Si+WcQBFSR>5Ltd-p0Es9B6ZS>1eQ8q z4olA4KFH#EB2{kJOA}SV9E67uxPjp!!?#v zp0g5I3U`otIaxA_ife$S#V;g8l4aOqWU2D$>J6Li(+d@K0?YXTnHG4pJ##|c^d*;n z0SoLCM3N*duq={!GAhxc+72x6{B@Gjlm)hKr-ehkXnw{=u)qTmBuRu19 zK-U}+(o73XS~E-^EYP8dgo!P1L=bbQXsnv+K3JIHZ%Kd*3*6L>Ie%%~D9P_djO z$!me43mo~pKn*Og-)9orv<3Rux}ycQnKR%aSm3~~Bv393Txr}6E%3MN)AvG`_}Kbm z&S>U5TEAV}S{nR&(+SIVCB?NN1XO;D6{79=FbV`J^ zuHBQ=^Sh_W0(E^SEroHZ8bU%O$7SFX?4Fq|zAC{^aLi$|kH8XjmBZrgGzeKP*JrN< z7L)Cyp3fq(z(7B9JU7V3lY~gJ^o~cC^>qrqz+!W1f11D&7R+I}bcXpQrAKlWJ{$C1 zAgSlGi0+>q-(!#m^D-@zgh;adD*;(Hsuq3)mX*;?Q#UuPvA{46%kY!TQ#4bG$MlAI zIT=st`7EL}7M{>*GO%n-BO#J3ofDDe%vNQ8V9~FC=_auF-Quv!In4ZuYG>U43b14; zy=tuIvxtng@Z?H7XDhcO36W$`OG1{IgG2C{ttB=y69pEpa1P7DbpoFJN*Bn)Y8V~_7PFP4 zp3frMp>wXC`vfdk?Ma9vOVu-+7n@*Rd^V`~iLcJqhFz^7p2PCYi}~5*_GCvq#lzz{ zspqqNV4n+gv$9zXEM1C7h$KsS3bI)JYl#EQ%i0^IKMO2rPdO|GFZ<)Xw7jts&p*?N zBK3S0(Y%yJ-n{}WfpH{6lI2}0vbdNG8Uic^Yp=Z&SmIJSEUopvN0x~Dh2gNqI&UTQ za$Wrqz(F!&w-@3c4}HSz%)?+#f2%xc zOp-9Ct7DJi_I)>U+Ep;8PwFJ4DRYYRWj+GhJB_WtoNjB9I9cYD-i5g`k2sA_U{1Az zNYpo()Bg2qb>W_qpDu}%Hm6}}j5%e@`_vy+RP;C!B&;ZrIrVBW5{Ew2cKtFir>#aL zq?zWFb!($0n9~+h5+*jMhntxfUG{!6T@@B)ggFV2VNS#6GOx%u`srOyFsDDPNs_$g zB)Y)aigT8MIsLwx#5QeCN@w<P^9z*;v?BlZ05 zDKe*HH7ygMd#2`*5UFud$Y;jo=(MyEWA2}gH7vxZ-4Aad^?Vl5 zJtx)eHF%$9)>aZC$&&IMS&Vb7b75W%{BtU;OT(__`;x=rW6!Yc3#h}lN)PWv>iI09 zd0Ba-3%-M|(_j)J$&!$XEFL?yb^?|QNu5p$EC=3jSpIu~`KGaRMhiS);b}Ih=d*~+ zX`9 z@=*X5^#~Fo$r6=~EQVfd&%*{aA@$A(*M>C~R4UV)RPvd#sa|&b^+zx#?>v$uVNM_7 zym0$Aym%!T%&G1bNomTQ0&X#Hl6f?vdkC1*kP;Fn%bZ5GKY~{_UPIRc%&ESTM17Mv zsgG8V19Q6cg+xl5(@Q38Ld(02vnZ@6kvS<;jzYj=z86)&oUBzyNHfjp z{mSd(z?{x?Bw=E6`rFD27iNq6|2~C<>8C*gWSG-e_BAM(n@{8a1kdy$N%ESL=mPs2 zJI)4k@)<~Cn>MG`uKUrPDvZwl4CdrIoCL~cPHRjw(VSZMjaPy$;ipG3CA)-04o=3e zb&q1eoW4vTG0iil0+-*5pnLk&>@gEMPGuRVdm5!Mw~<5tJxL$d`q!3w8|(SqQ?#pv zYSsqBxY)KOA(G>AhZz^OF2PTMWz*3s#|4%@KXO=VPci@Rz1g8Zo)$6o6shO4h<3G` zzV$l5QXW7;Bw508agIG+_?!h6kDDV(1(w_@4vYFi=7a^!&L13MURHEJ1n5;;*e)1}t~2FVEWEu&X84a#$3s8KXU{8R-fvA!|uJpGCCBQh#oX z|3b*$L_#E40`rljAos*AVEH2?beF*L;BOAg#39}Bn?_a8h_}Gv6;A5;EF#;8F;doq zHTKIR5+ccR{spp}&EQu`=D+HEXyOhMwYxddF6It}M1{?zx#df4#PL|R7_OZa? z+m(b!viKDsi}r-*1+c@N_;kIWTf-Vl+pVN6-r>6az`Vfo-aWHda2?xP-DnJuypA=6 Y%ykU)DuCXc(2m43uQxx~Gr#@+02J@YLjV8( literal 0 HcmV?d00001 diff --git a/logs/self_tuning/ademamix_golden/study_0/imagenet_resnet_pytorch/trial_1/flags_0.json b/logs/self_tuning/ademamix_golden/study_0/imagenet_resnet_pytorch/trial_1/flags_0.json new file mode 100644 index 00000000..50000741 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/imagenet_resnet_pytorch/trial_1/flags_0.json @@ -0,0 +1,63 @@ +{ + "logtostderr": false, + "alsologtostderr": false, + "log_dir": "", + "v": 0, + "verbosity": 0, + "logger_levels": {}, + "stderrthreshold": "fatal", + "showprefixforinfo": true, + "run_with_pdb": false, + "pdb_post_mortem": false, + "pdb": false, + "run_with_profiling": false, + "profile_file": null, + "use_cprofile_for_profiling": true, + "only_check_args": false, + "op_conversion_fallback_to_while_loop": true, + "delta_threshold": 0.5, + "tt_check_filter": false, + "tt_single_core_summaries": false, + "runtime_oom_exit": true, + "hbm_oom_exit": true, + "test_srcdir": "", + "test_tmpdir": "/tmp/absl_testing", + "test_random_seed": 301, + "test_randomize_ordering_seed": "", + "xml_output_file": "", + "experimental_orbax_use_distributed_process_id": false, + "experimental_orbax_use_distributed_barrier": false, + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "workload": "imagenet_resnet", + "tuning_ruleset": "self", + "tuning_search_space": null, + "num_tuning_trials": 1, + "data_dir": "/data/imagenet/pytorch", + "imagenet_v2_data_dir": "/data/imagenet/pytorch", + "librispeech_tokenizer_vocab_path": "", + "framework": "pytorch", + "torch_compile": true, + "experiment_dir": "/experiment_runs", + "experiment_name": "submissions_a100/ademamix_golden/study_0", + "save_checkpoints": false, + "save_intermediate_checkpoints": true, + "resume_last_run": null, + "append_timestamp": false, + "use_wandb": false, + "profile": false, + "max_global_steps": null, + "overwrite": true, + "hparam_start_index": null, + "hparam_end_index": null, + "rng_seed": -1820589156, + "set_pytorch_max_split_size": false, + "pytorch_eval_num_workers": 0, + "capture_jax_trace": false, + "skip_evals": false, + "?": false, + "help": false, + "helpshort": false, + "helpfull": false, + "helpxml": false, + "tfds_debug_list_dir": false +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_0/imagenet_resnet_pytorch/trial_1/measurements.csv b/logs/self_tuning/ademamix_golden/study_0/imagenet_resnet_pytorch/trial_1/measurements.csv new file mode 100644 index 00000000..e90a659b --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/imagenet_resnet_pytorch/trial_1/measurements.csv @@ -0,0 +1,371 @@ +global_step,grad_norm,loss,train/accuracy,train/loss,validation/accuracy,validation/loss,validation/num_examples,test/accuracy,test/loss,test/num_examples,score,total_duration,accumulated_submission_time,accumulated_eval_time,accumulated_logging_time,preemption_count +0,0.4999991953372955,6.928142070770264,,,,,,,,,,,,,, +1,,,0.0011957908163265,6.911289137236926,0.00114,6.91032625,50000.0,0.0016,6.91218828125,10000.0,121.5451877117157,324.81873393058777,121.5451877117157,202.6214411258697,0.0,0.0 +1,0.4999992549419403,6.931043148040772,,,,,,,,,,,,,, +2,0.4999992847442627,6.921487808227539,,,,,,,,,,,,,, +3,0.4999992549419403,6.932662487030029,,,,,,,,,,,,,, +4,0.4999992251396179,6.929649353027344,,,,,,,,,,,,,, +5,0.4999992251396179,6.922558307647705,,,,,,,,,,,,,, +6,0.4999992251396179,6.9322052001953125,,,,,,,,,,,,,, +7,0.4999992549419403,6.919102668762207,,,,,,,,,,,,,, +8,0.4999992847442627,6.925088405609131,,,,,,,,,,,,,, +9,0.4999992847442627,6.918175220489502,,,,,,,,,,,,,, +10,0.4999992251396179,6.92986536026001,,,,,,,,,,,,,, +11,0.4999991953372955,6.926110744476318,,,,,,,,,,,,,, +12,0.4999992251396179,6.920243263244629,,,,,,,,,,,,,, +13,0.4999992549419403,6.925408840179443,,,,,,,,,,,,,, +14,0.4999992847442627,6.926761627197266,,,,,,,,,,,,,, +15,0.4999992847442627,6.926137924194336,,,,,,,,,,,,,, +16,0.4999993145465851,6.932054996490479,,,,,,,,,,,,,, +17,0.4999992251396179,6.932072162628174,,,,,,,,,,,,,, +18,0.4999993145465851,6.929481029510498,,,,,,,,,,,,,, +19,0.4999992251396179,6.925643444061279,,,,,,,,,,,,,, +20,0.4999992251396179,6.935122489929199,,,,,,,,,,,,,, +21,0.4999992549419403,6.919253349304199,,,,,,,,,,,,,, +22,0.4999992549419403,6.920987129211426,,,,,,,,,,,,,, +23,0.4999992251396179,6.919617652893066,,,,,,,,,,,,,, +24,0.4999991953372955,6.914258003234863,,,,,,,,,,,,,, +25,0.4999992251396179,6.930021286010742,,,,,,,,,,,,,, +26,0.4999993145465851,6.935941696166992,,,,,,,,,,,,,, +27,0.4999992847442627,6.933531761169434,,,,,,,,,,,,,, +28,0.4999992549419403,6.916841983795166,,,,,,,,,,,,,, +29,0.4999992549419403,6.920168399810791,,,,,,,,,,,,,, +30,0.4999992549419403,6.931833744049072,,,,,,,,,,,,,, +31,0.4999992549419403,6.931797981262207,,,,,,,,,,,,,, +32,0.4999991655349731,6.932475090026856,,,,,,,,,,,,,, +33,0.4999993145465851,6.922188758850098,,,,,,,,,,,,,, +34,0.4999992549419403,6.923115253448486,,,,,,,,,,,,,, +35,0.4999992251396179,6.92905330657959,,,,,,,,,,,,,, +36,0.4999992251396179,6.930643081665039,,,,,,,,,,,,,, +37,0.4999992251396179,6.9040117263793945,,,,,,,,,,,,,, +38,0.4999992549419403,6.92165994644165,,,,,,,,,,,,,, +39,0.4999992549419403,6.929865837097168,,,,,,,,,,,,,, +40,0.4999993145465851,6.915813446044922,,,,,,,,,,,,,, +41,0.4999992549419403,6.915447235107422,,,,,,,,,,,,,, +42,0.4999992251396179,6.919086933135986,,,,,,,,,,,,,, +43,0.4999992549419403,6.920093059539795,,,,,,,,,,,,,, +44,0.4999992847442627,6.920204162597656,,,,,,,,,,,,,, +45,0.4999992549419403,6.910027980804443,,,,,,,,,,,,,, +46,0.4999992847442627,6.915291786193848,,,,,,,,,,,,,, +47,0.4999992549419403,6.911962032318115,,,,,,,,,,,,,, +48,0.4999992847442627,6.930974006652832,,,,,,,,,,,,,, +49,0.4999992251396179,6.916858673095703,,,,,,,,,,,,,, +50,0.4999992251396179,6.9207611083984375,,,,,,,,,,,,,, +51,0.4999991953372955,6.926780700683594,,,,,,,,,,,,,, +52,0.4999991953372955,6.924049377441406,,,,,,,,,,,,,, +53,0.4999992549419403,6.916375160217285,,,,,,,,,,,,,, +54,0.4999992549419403,6.916347503662109,,,,,,,,,,,,,, +55,0.4999992549419403,6.913029670715332,,,,,,,,,,,,,, +56,0.4999992549419403,6.906192302703857,,,,,,,,,,,,,, +57,0.4999993145465851,6.925858497619629,,,,,,,,,,,,,, +58,0.4999992549419403,6.920598030090332,,,,,,,,,,,,,, +59,0.4999992847442627,6.911448955535889,,,,,,,,,,,,,, +60,0.4999992847442627,6.911592960357666,,,,,,,,,,,,,, +61,0.4999992251396179,6.921866893768311,,,,,,,,,,,,,, +62,0.4999992847442627,6.928030014038086,,,,,,,,,,,,,, +63,0.4999992847442627,6.919266700744629,,,,,,,,,,,,,, +64,0.4999992549419403,6.907110214233398,,,,,,,,,,,,,, +65,0.4999992847442627,6.9260687828063965,,,,,,,,,,,,,, +66,0.4999992549419403,6.917031288146973,,,,,,,,,,,,,, +67,0.4999992251396179,6.920480251312256,,,,,,,,,,,,,, +68,0.4999992549419403,6.91558837890625,,,,,,,,,,,,,, +69,0.4999991953372955,6.90995979309082,,,,,,,,,,,,,, +70,0.4999992251396179,6.91126012802124,,,,,,,,,,,,,, +71,0.4999992549419403,6.919085502624512,,,,,,,,,,,,,, +72,0.4999993443489074,6.901516914367676,,,,,,,,,,,,,, +73,0.4999992251396179,6.916186332702637,,,,,,,,,,,,,, +74,0.4999992251396179,6.906358242034912,,,,,,,,,,,,,, +75,0.4999991953372955,6.91094446182251,,,,,,,,,,,,,, +76,0.4999992847442627,6.909159183502197,,,,,,,,,,,,,, +77,0.4999992847442627,6.917328357696533,,,,,,,,,,,,,, +78,0.4999992549419403,6.902870655059815,,,,,,,,,,,,,, +79,0.4999992549419403,6.9043049812316895,,,,,,,,,,,,,, +80,0.4999992549419403,6.907343864440918,,,,,,,,,,,,,, +81,0.4999992847442627,6.911849021911621,,,,,,,,,,,,,, +82,0.4999991953372955,6.9111857414245605,,,,,,,,,,,,,, +83,0.4999992549419403,6.905432224273682,,,,,,,,,,,,,, +84,0.4999992549419403,6.908701419830322,,,,,,,,,,,,,, +85,0.4999992549419403,6.914257049560547,,,,,,,,,,,,,, +86,0.4999992251396179,6.90020227432251,,,,,,,,,,,,,, +87,0.4999992549419403,6.900778770446777,,,,,,,,,,,,,, +88,0.4999992251396179,6.900518417358398,,,,,,,,,,,,,, +89,0.4999992549419403,6.896376609802246,,,,,,,,,,,,,, +90,0.4999992549419403,6.908195495605469,,,,,,,,,,,,,, +91,0.4999991655349731,6.89870548248291,,,,,,,,,,,,,, +92,0.4999992847442627,6.899560451507568,,,,,,,,,,,,,, +93,0.4999992549419403,6.902437210083008,,,,,,,,,,,,,, +94,0.4999992847442627,6.905853271484375,,,,,,,,,,,,,, +95,0.4999992549419403,6.890774726867676,,,,,,,,,,,,,, +96,0.4999992549419403,6.898809909820557,,,,,,,,,,,,,, +97,0.4999992549419403,6.890564441680908,,,,,,,,,,,,,, +98,0.4999992847442627,6.896762847900391,,,,,,,,,,,,,, +99,0.4999992251396179,6.88991641998291,,,,,,,,,,,,,, +100,0.4999992847442627,6.901190757751465,,,,,,,,,,,,,, +500,0.499999463558197,6.433737754821777,,,,,,,,,,,,,, +1000,0.4999997019767761,5.894944190979004,,,,,,,,,,,,,, +1500,0.4999998211860657,5.484642028808594,,,,,,,,,,,,,, +1911,,,0.0758330676020408,5.683907645089286,0.08448,5.369495,50000.0,0.0581,5.43460625,10000.0,2115.945389509201,2448.4243199825287,2115.945389509201,329.35181760787964,0.0297186374664306,0.0 +2000,0.499999850988388,5.128137588500977,,,,,,,,,,,,,, +2500,0.4999999105930328,4.872332572937012,,,,,,,,,,,,,, +3000,0.4999999105930328,4.495009422302246,,,,,,,,,,,,,, +3500,0.4999999105930328,4.14410400390625,,,,,,,,,,,,,, +4000,0.4999999105930328,4.109152793884277,,,,,,,,,,,,,, +4500,0.4999998807907104,3.507976531982422,,,,,,,,,,,,,, +4679,,,0.2529695471938775,4.555435180664063,0.2422,4.4598834375,50000.0,0.1978,4.166992578125,10000.0,4113.932512521744,4566.972930669785,4113.932512521744,446.6659274101257,0.0852634906768798,0.0 +5000,0.4999999701976776,3.5122036933898926,,,,,,,,,,,,,, +5500,0.4999999105930328,3.2564268112182617,,,,,,,,,,,,,, +6000,0.4999999403953552,3.065365791320801,,,,,,,,,,,,,, +6500,0.4999998211860657,2.8919005393981934,,,,,,,,,,,,,, +7000,0.4999998807907104,2.775651693344116,,,,,,,,,,,,,, +7500,0.4999998807907104,2.6904726028442383,,,,,,,,,,,,,, +7813,,,0.4213966836734694,4.301740373883929,0.41084,3.6116853125,50000.0,0.3349,3.22026875,10000.0,6106.99627327919,6668.810752868652,6106.99627327919,551.2657015323639,0.1217114925384521,0.0 +8000,0.4999999105930328,2.614917516708374,,,,,,,,,,,,,, +8500,0.4999998807907104,2.4274580478668213,,,,,,,,,,,,,, +9000,0.4999998807907104,2.348928928375244,,,,,,,,,,,,,, +9500,0.4999997317790985,2.2993178367614746,,,,,,,,,,,,,, +10000,0.499999850988388,2.167365074157715,,,,,,,,,,,,,, +10500,0.4999997913837433,2.209725379943848,,,,,,,,,,,,,, +10908,,,0.5089684311224489,3.690275542590083,0.48904,3.393254375,50000.0,0.4208,2.76146171875,10000.0,8100.409832239151,8796.185854911804,8100.409832239151,681.1285533905029,0.1685569286346435,0.0 +11000,0.4999997615814209,2.081854820251465,,,,,,,,,,,,,, +11500,0.499999850988388,1.994966983795166,,,,,,,,,,,,,, +12000,0.4999998807907104,1.9386489391326904,,,,,,,,,,,,,, +12500,0.4999997317790985,2.013587713241577,,,,,,,,,,,,,, +13000,0.4999997019767761,1.920423030853272,,,,,,,,,,,,,, +13500,0.4999997913837433,1.8788251876831048,,,,,,,,,,,,,, +13975,,,0.5401785714285714,3.8735943229831,0.49856,3.5946475,50000.0,0.4468,2.5407837890625,10000.0,10093.10680270195,10894.439922094343,10093.10680270195,782.3462271690369,0.3399162292480469,0.0 +14000,0.4999997317790985,1.88968825340271,,,,,,,,,,,,,, +14500,0.4999997615814209,1.8500571250915527,,,,,,,,,,,,,, +15000,0.4999997913837433,1.832858085632324,,,,,,,,,,,,,, +15500,0.4999997615814209,1.564108967781067,,,,,,,,,,,,,, +16000,0.4999997019767761,1.7138302326202393,,,,,,,,,,,,,, +16500,0.4999997019767761,1.7364306449890137,,,,,,,,,,,,,, +17000,0.4999997317790985,1.643286943435669,,,,,,,,,,,,,, +17094,,,0.5809351084183674,3.806796326929209,0.55234,3.24312875,50000.0,0.4778,2.466045703125,10000.0,12085.535720586777,13025.37265920639,12085.535720586777,916.060443162918,0.5296454429626465,0.0 +17500,0.4999997019767761,1.649580001831055,,,,,,,,,,,,,, +18000,0.4999997317790985,1.6507946252822876,,,,,,,,,,,,,, +18500,0.4999997913837433,1.6435880661010742,,,,,,,,,,,,,, +19000,0.4999997019767761,1.5785846710205078,,,,,,,,,,,,,, +19500,0.4999996423721313,1.4822638034820557,,,,,,,,,,,,,, +20000,0.4999997019767761,1.515758752822876,,,,,,,,,,,,,, +20017,,,0.5847417091836735,4.2652899294483415,0.57624,3.0346875,50000.0,0.4969,2.292778125,10000.0,14080.839374542236,15124.489433050156,14080.839374542236,1014.7383263111116,0.7073013782501221,0.0 +20500,0.499999850988388,1.581475019454956,,,,,,,,,,,,,, +21000,0.4999997317790985,1.5577077865600586,,,,,,,,,,,,,, +21500,0.4999997913837433,1.57105553150177,,,,,,,,,,,,,, +22000,0.4999997913837433,1.5753540992736816,,,,,,,,,,,,,, +22500,0.4999997317790985,1.5915396213531494,,,,,,,,,,,,,, +23000,0.4999996125698089,1.5011811256408691,,,,,,,,,,,,,, +23241,,,0.6491749043367347,3.424844547193877,0.57928,3.2999325,50000.0,0.5222,2.24196796875,10000.0,16076.302437067032,17248.351852178574,16076.302437067032,1136.4618639945984,1.0467360019683838,0.0 +23500,0.4999996721744537,1.433100938796997,,,,,,,,,,,,,, +24000,0.4999997615814209,1.3706063032150269,,,,,,,,,,,,,, +24500,0.4999997913837433,1.432210922241211,,,,,,,,,,,,,, +25000,0.4999996721744537,1.4835833311080933,,,,,,,,,,,,,, +25500,0.4999997019767761,1.3947951793670654,,,,,,,,,,,,,, +26000,0.4999996721744537,1.3705281019210815,,,,,,,,,,,,,, +26171,,,0.6524035395408163,3.653707387496014,0.58686,3.1516565625,50000.0,0.5141,2.249761328125,10000.0,18069.75966453552,19362.27117872238,18069.75966453552,1251.8468353748322,1.3355841636657717,0.0 +26500,0.4999997019767761,1.4225207567214966,,,,,,,,,,,,,, +27000,0.4999997317790985,1.3475582599639893,,,,,,,,,,,,,, +27500,0.4999996423721313,1.3875772953033447,,,,,,,,,,,,,, +28000,0.4999997019767761,1.2693488597869873,,,,,,,,,,,,,, +28500,0.4999997019767761,1.341657042503357,,,,,,,,,,,,,, +29000,0.4999995827674866,1.3954020738601685,,,,,,,,,,,,,, +29427,,,0.6569674744897959,3.933512784996811,0.5849,3.3899871875,50000.0,0.5272,2.18953828125,10000.0,20062.942791700363,21481.680223941803,20062.942791700363,1371.63134431839,1.625967264175415,0.0 +29500,0.4999996423721313,1.2952405214309692,,,,,,,,,,,,,, +30000,0.4999995827674866,1.2287832498550415,,,,,,,,,,,,,, +30500,0.4999996423721313,1.2885663509368896,,,,,,,,,,,,,, +31000,0.4999996423721313,1.202884554862976,,,,,,,,,,,,,, +31500,0.4999996423721313,1.248686909675598,,,,,,,,,,,,,, +32000,0.4999996721744537,1.2425408363342283,,,,,,,,,,,,,, +32376,,,0.6981425382653061,3.257666451590402,0.60376,3.2073090625,50000.0,0.5364,2.1774421875,10000.0,22062.18269753456,23603.81263756752,22062.18269753456,1489.5095262527466,1.98438024520874,0.0 +32500,0.4999997019767761,1.3965626955032349,,,,,,,,,,,,,, +33000,0.4999997019767761,1.310892939567566,,,,,,,,,,,,,, +33500,0.4999996125698089,1.3015377521514893,,,,,,,,,,,,,, +34000,0.4999995529651642,1.1992563009262085,,,,,,,,,,,,,, +34500,0.4999996423721313,1.188756823539734,,,,,,,,,,,,,, +35000,0.4999996423721313,1.1084036827087402,,,,,,,,,,,,,, +35500,0.4999995529651642,1.13248872756958,,,,,,,,,,,,,, +35628,,,0.7033641581632653,3.4684765874123085,0.59992,3.4122078125,50000.0,0.5405,2.2027775390625,10000.0,24057.024208545685,25719.10164737701,24057.024208545685,1604.0221438407898,2.28261399269104,0.0 +36000,0.4999996423721313,1.2175142765045166,,,,,,,,,,,,,, +36500,0.4999996423721313,1.2100846767425537,,,,,,,,,,,,,, +37000,0.4999996423721313,1.26710844039917,,,,,,,,,,,,,, +37500,0.4999995827674866,1.234635353088379,,,,,,,,,,,,,, +38000,0.4999996721744537,1.1898536682128906,,,,,,,,,,,,,, +38500,0.4999995529651642,1.1988143920898438,,,,,,,,,,,,,, +38590,,,0.7145448022959183,3.2748126594387754,0.58636,3.774430625,50000.0,0.5371,2.18135625,10000.0,26055.084835529327,27841.924547433853,26055.084835529327,1723.3271679878235,2.560221672058105,0.0 +39000,0.4999996125698089,1.1588006019592283,,,,,,,,,,,,,, +39500,0.4999996125698089,1.1351609230041504,,,,,,,,,,,,,, +40000,0.4999996423721313,1.1697349548339844,,,,,,,,,,,,,, +40500,0.4999996423721313,1.1279923915863037,,,,,,,,,,,,,, +41000,0.4999996423721313,1.0951553583145142,,,,,,,,,,,,,, +41500,0.4999995827674866,1.0421874523162842,,,,,,,,,,,,,, +41831,,,0.7114158163265306,3.6007372797751915,0.60598,3.3401575,50000.0,0.5336,2.324109765625,10000.0,28047.74717950821,29952.44298005104,28047.74717950821,1835.4817848205569,2.834051609039306,0.0 +42000,0.4999995529651642,1.1658363342285156,,,,,,,,,,,,,, +42500,0.4999996125698089,1.153024673461914,,,,,,,,,,,,,, +43000,0.4999996423721313,1.0882905721664429,,,,,,,,,,,,,, +43500,0.4999994933605194,1.0124220848083496,,,,,,,,,,,,,, +44000,0.4999996423721313,1.1841483116149902,,,,,,,,,,,,,, +44500,0.4999995827674866,1.1365764141082764,,,,,,,,,,,,,, +44812,,,0.6891940369897959,4.2108745964206,0.60408,3.409203125,50000.0,0.5384,2.2525880859375,10000.0,30044.303684949875,32077.29454255104,30044.303684949875,1959.2915556430817,3.106719970703125,0.0 +45000,0.4999995231628418,1.1066771745681765,,,,,,,,,,,,,, +45500,0.4999996125698089,1.0417931079864502,,,,,,,,,,,,,, +46000,0.4999996423721313,1.0955127477645874,,,,,,,,,,,,,, +46500,0.4999996125698089,1.1653863191604614,,,,,,,,,,,,,, +47000,0.4999997317790985,1.165544867515564,,,,,,,,,,,,,, +47500,0.4999995827674866,1.1173263788223269,,,,,,,,,,,,,, +48000,0.4999996125698089,1.0502996444702148,,,,,,,,,,,,,, +48029,,,0.7480269451530612,3.264349489795918,0.61424,3.2800534375,50000.0,0.5376,2.3202591796875,10000.0,32036.41220641136,34185.70292234421,32036.41220641136,2069.0723991394043,3.439774990081787,0.0 +48500,0.4999996125698089,1.082904815673828,,,,,,,,,,,,,, +49000,0.4999996423721313,0.979789972305298,,,,,,,,,,,,,, +49500,0.4999996125698089,1.0127242803573608,,,,,,,,,,,,,, +50000,0.4999996721744537,1.0346522331237793,,,,,,,,,,,,,, +50500,0.4999995529651642,1.0281494855880735,,,,,,,,,,,,,, +51000,0.4999995529651642,1.0303236246109009,,,,,,,,,,,,,, +51025,,,0.7228156887755102,4.107982090541294,0.60628,3.646350625,50000.0,0.5283,2.405058984375,10000.0,34034.62011766434,36309.644293785095,34034.62011766434,2189.3234827518463,3.695246696472168,0.0 +51500,0.4999995529651642,0.9283828735351562,,,,,,,,,,,,,, +52000,0.4999995827674866,0.9858921766281128,,,,,,,,,,,,,, +52500,0.4999996721744537,1.1044315099716189,,,,,,,,,,,,,, +53000,0.4999996125698089,0.9239526987075806,,,,,,,,,,,,,, +53500,0.4999996125698089,1.0212769508361816,,,,,,,,,,,,,, +54000,0.4999995827674866,1.0361671447753906,,,,,,,,,,,,,, +54173,,,0.7489038584183674,3.67700600137516,0.61914,3.423630625,50000.0,0.5371,2.37350625,10000.0,36030.59577226639,38411.74111676216,36030.59577226639,2291.069113969803,3.971768617630005,0.0 +54500,0.4999996125698089,1.0259175300598145,,,,,,,,,,,,,, +55000,0.4999995827674866,1.027035117149353,,,,,,,,,,,,,, +55500,0.4999995827674866,0.8960285186767578,,,,,,,,,,,,,, +56000,0.4999996125698089,1.0296629667282104,,,,,,,,,,,,,, +56500,0.4999994933605194,0.9508575201034546,,,,,,,,,,,,,, +57000,0.4999995827674866,0.9652302265167236,,,,,,,,,,,,,, +57207,,,0.7190489477040817,4.503145178970025,0.60846,3.6531325,50000.0,0.5381,2.3990828125,10000.0,38026.40539455414,40536.13014054298,38026.40539455414,2414.7549934387207,4.314009189605713,0.0 +57500,0.4999996125698089,0.9689395427703856,,,,,,,,,,,,,, +58000,0.4999996125698089,0.9899077415466307,,,,,,,,,,,,,, +58500,0.4999995529651642,0.9784048795700072,,,,,,,,,,,,,, +59000,0.4999995529651642,0.9653661251068116,,,,,,,,,,,,,, +59500,0.4999996125698089,0.9897361397743224,,,,,,,,,,,,,, +60000,0.4999995827674866,1.0230456590652466,,,,,,,,,,,,,, +60240,,,0.7339365433673469,4.250097780811544,0.60516,3.680009375,50000.0,0.5419,2.3658654296875,10000.0,40019.40243244171,42635.11920881271,40019.40243244171,2515.279800891876,4.556050777435303,0.0 +60500,0.4999996125698089,0.9806970357894896,,,,,,,,,,,,,, +61000,0.4999995529651642,1.0078173875808716,,,,,,,,,,,,,, +61500,0.4999994933605194,0.8938435316085815,,,,,,,,,,,,,, +62000,0.4999995827674866,0.999146044254303,,,,,,,,,,,,,, +62500,0.4999995529651642,0.9260661602020264,,,,,,,,,,,,,, +63000,0.4999996125698089,0.988490343093872,,,,,,,,,,,,,, +63355,,,0.7662228954081632,3.4690078423947703,0.60764,3.729314375,50000.0,0.5399,2.441900390625,10000.0,42017.28895163536,44756.50366902352,42017.28895163536,2632.641822338104,4.827040195465088,0.0 +63500,0.4999995529651642,0.8697692155838013,,,,,,,,,,,,,, +64000,0.4999995529651642,0.8287724256515503,,,,,,,,,,,,,, +64500,0.4999996125698089,0.9018519520759584,,,,,,,,,,,,,, +65000,0.4999996125698089,0.964820146560669,,,,,,,,,,,,,, +65500,0.4999995827674866,0.8578883409500122,,,,,,,,,,,,,, +66000,0.4999996423721313,0.9383552074432372,,,,,,,,,,,,,, +66260,,,0.7411910076530612,4.541734967912946,0.61386,3.7965825,50000.0,0.5465,2.424560546875,10000.0,44011.472423791885,46843.58191680908,44011.472423791885,2721.444454908371,5.059232711791992,0.0 +66500,0.4999995827674866,0.8367151618003845,,,,,,,,,,,,,, +67000,0.4999995827674866,0.8786455988883972,,,,,,,,,,,,,, +67500,0.4999996125698089,0.8676085472106934,,,,,,,,,,,,,, +68000,0.4999996423721313,0.8985345363616943,,,,,,,,,,,,,, +68500,0.4999995827674866,0.9546284675598145,,,,,,,,,,,,,, +69000,0.4999995827674866,0.8896344900131226,,,,,,,,,,,,,, +69453,,,0.7394571109693877,4.600998236208546,0.60794,3.766720625,50000.0,0.5419,2.48736328125,10000.0,46004.137469530106,48965.33098864555,46004.137469530106,2844.452032327652,5.329614877700806,0.0 +69500,0.4999995529651642,0.9002063274383545,,,,,,,,,,,,,, +70000,0.4999995529651642,0.8381842374801636,,,,,,,,,,,,,, +70500,0.4999996125698089,0.8423949480056763,,,,,,,,,,,,,, +71000,0.4999996423721313,0.792049765586853,,,,,,,,,,,,,, +71500,0.4999995529651642,0.8187052011489868,,,,,,,,,,,,,, +72000,0.4999996125698089,0.9215567111968994,,,,,,,,,,,,,, +72390,,,0.7686344068877551,4.249235192123725,0.61002,3.8339628125,50000.0,0.5405,2.52101484375,10000.0,47997.16594219208,51085.02428627014,47997.16594219208,2965.7500581741333,5.646993637084961,0.0 +72500,0.4999995529651642,0.9159996509552002,,,,,,,,,,,,,, +73000,0.4999996125698089,0.8868779540061951,,,,,,,,,,,,,, +73500,0.4999996423721313,0.9126135110855104,,,,,,,,,,,,,, +74000,0.4999995529651642,0.8992960453033447,,,,,,,,,,,,,, +74500,0.4999995827674866,0.9111217260360718,,,,,,,,,,,,,, +75000,0.4999996125698089,0.8323758244514465,,,,,,,,,,,,,, +75500,0.4999997019767761,0.8428764939308167,,,,,,,,,,,,,, +75606,,,0.7893813775510204,3.65380859375,0.60568,3.93482125,50000.0,0.5424,2.534219921875,10000.0,49989.45276546478,53195.83649134636,49989.45276546478,3077.8687732219696,5.9023377895355225,0.0 +76000,0.4999995827674866,0.8221288323402405,,,,,,,,,,,,,, +76500,0.4999995827674866,0.8143126964569092,,,,,,,,,,,,,, +77000,0.4999996125698089,0.8437207937240601,,,,,,,,,,,,,, +77500,0.4999995827674866,0.7716533541679382,,,,,,,,,,,,,, +78000,0.4999996125698089,0.8013293147087097,,,,,,,,,,,,,, +78500,0.4999995529651642,0.8609068393707275,,,,,,,,,,,,,, +78568,,,0.8084143813775511,3.43436167191486,0.60452,4.179504375,50000.0,0.5439,2.54808125,10000.0,51982.49979019165,55311.74465513229,51982.49979019165,3196.0724246501923,6.18151068687439,0.0 +79000,0.4999995231628418,0.7431334257125854,,,,,,,,,,,,,, +79500,0.4999995529651642,0.8108168244361877,,,,,,,,,,,,,, +80000,0.4999995827674866,0.8785037398338318,,,,,,,,,,,,,, +80500,0.4999995529651642,0.8462681770324707,,,,,,,,,,,,,, +81000,0.4999995529651642,0.793304443359375,,,,,,,,,,,,,, +81500,0.4999995529651642,0.7268357872962952,,,,,,,,,,,,,, +81755,,,0.7738360969387755,4.3157329948581,0.60878,3.9008559375,50000.0,0.5395,2.718340625,10000.0,53974.51514530182,57420.78761053085,53974.51514530182,3306.6501145362854,6.569916725158691,0.0 +82000,0.4999995529651642,0.7721753716468811,,,,,,,,,,,,,, +82500,0.499999463558197,0.8540744781494141,,,,,,,,,,,,,, +83000,0.4999995231628418,0.8112013339996338,,,,,,,,,,,,,, +83500,0.4999995529651642,0.8165156245231628,,,,,,,,,,,,,, +84000,0.4999996125698089,0.7585240602493286,,,,,,,,,,,,,, +84500,0.4999995529651642,0.8467531204223633,,,,,,,,,,,,,, +84769,,,0.7844985650510204,4.187828219666773,0.5956,4.42532375,50000.0,0.5363,2.6479666015625,10000.0,55967.39384794235,59543.1830201149,55967.39384794235,3430.871250152588,6.843689918518066,0.0 +85000,0.4999995827674866,0.7381881475448608,,,,,,,,,,,,,, +85500,0.4999996423721313,0.8018828630447388,,,,,,,,,,,,,, +86000,0.4999995231628418,0.7692781090736389,,,,,,,,,,,,,, +86500,0.4999996125698089,0.7544387578964233,,,,,,,,,,,,,, +87000,0.4999995231628418,0.738789975643158,,,,,,,,,,,,,, +87500,0.4999996125698089,0.7753422260284424,,,,,,,,,,,,,, +87863,,,0.8123604910714286,3.558631741270727,0.60924,4.02082625,50000.0,0.5439,2.617235546875,10000.0,57960.61058306694,61643.68226647377,57960.61058306694,3533.468069791794,7.181390047073364,0.0 +88000,0.4999995231628418,0.771352231502533,,,,,,,,,,,,,, +88500,0.4999996423721313,0.7897124290466309,,,,,,,,,,,,,, +89000,0.4999995827674866,0.7128024101257324,,,,,,,,,,,,,, +89500,0.4999995231628418,0.6640494465827942,,,,,,,,,,,,,, +90000,0.4999995529651642,0.7842349410057068,,,,,,,,,,,,,, +90500,0.4999996423721313,0.8447709083557129,,,,,,,,,,,,,, +90929,,,0.8195352359693877,3.606179373604911,0.60892,4.214265,50000.0,0.5419,2.7006576171875,10000.0,59953.10913348198,63768.584854602814,59953.10913348198,3660.716259479522,7.5574729442596436,0.0 +91000,0.4999996125698089,0.7475289106369019,,,,,,,,,,,,,, +91500,0.4999995827674866,0.7761568427085876,,,,,,,,,,,,,, +92000,0.4999995529651642,0.6633009314537048,,,,,,,,,,,,,, +92500,0.4999995827674866,0.7319304943084717,,,,,,,,,,,,,, +93000,0.4999995231628418,0.7391221523284912,,,,,,,,,,,,,, +93500,0.4999996125698089,0.7180588245391846,,,,,,,,,,,,,, +93885,,,0.7375438456632653,6.002884845344388,0.61222,4.09152,50000.0,0.5458,2.68063203125,10000.0,61947.415028095245,65866.71648287773,61947.415028095245,3758.870266914368,7.838247060775757,0.0 +94000,0.4999994337558746,0.63383549451828,,,,,,,,,,,,,, +94500,0.4999996423721313,0.7717768549919128,,,,,,,,,,,,,, +95000,0.4999996423721313,0.7413055896759033,,,,,,,,,,,,,, +95500,0.4999995231628418,0.6990301609039307,,,,,,,,,,,,,, +96000,0.4999995529651642,0.6447580456733704,,,,,,,,,,,,,, +96500,0.4999995827674866,0.7005155682563782,,,,,,,,,,,,,, +97000,0.4999995827674866,0.6522864103317261,,,,,,,,,,,,,, +97041,,,0.7970145089285714,4.257398644272162,0.60598,4.1292421875,50000.0,0.5379,2.7065611328125,10000.0,63939.98579668999,67980.07772994041,63939.98579668999,3873.47101354599,8.128909587860107,0.0 +97500,0.4999995827674866,0.7563415169715881,,,,,,,,,,,,,, +98000,0.4999995231628418,0.6707614064216614,,,,,,,,,,,,,, +98500,0.4999996125698089,0.7597394585609436,,,,,,,,,,,,,, +99000,0.4999995529651642,0.63887619972229,,,,,,,,,,,,,, +99500,0.4999996423721313,0.6333047151565552,,,,,,,,,,,,,, +99969,,,0.7916533801020408,4.6550697793765945,0.60732,4.32330125,50000.0,0.537,2.738551953125,10000.0,65939.92808747292,70100.81326365471,65939.92808747292,3989.0768382549286,8.517204523086548,0.0 +100000,0.4999995827674866,0.639176607131958,,,,,,,,,,,,,, +100500,0.4999995529651642,0.6653611063957214,,,,,,,,,,,,,, +101000,0.4999996423721313,0.7078526616096497,,,,,,,,,,,,,, +101500,0.4999996125698089,0.6324746012687683,,,,,,,,,,,,,, +102000,0.4999996125698089,0.7552664875984192,,,,,,,,,,,,,, +102500,0.4999996125698089,0.7002390027046204,,,,,,,,,,,,,, +103000,0.4999996125698089,0.6737380027770996,,,,,,,,,,,,,, +103202,,,0.8127391581632653,4.210806710379464,0.61348,4.36987,50000.0,0.5385,2.759041015625,10000.0,67932.3721754551,72218.91586017609,67932.3721754551,4108.448180913925,8.829198837280273,0.0 +103500,0.4999995827674866,0.7045892477035522,,,,,,,,,,,,,, +104000,0.4999995529651642,0.6541073322296143,,,,,,,,,,,,,, +104500,0.4999995231628418,0.6508737206459045,,,,,,,,,,,,,, +105000,0.4999995827674866,0.7367428541183472,,,,,,,,,,,,,, +105500,0.4999996125698089,0.69749915599823,,,,,,,,,,,,,, +106000,0.4999995827674866,0.6589133143424988,,,,,,,,,,,,,, +106137,,,0.8254145408163265,3.80894812759088,0.60976,4.3623375,50000.0,0.5392,2.8169044921875,10000.0,69925.38329291344,74338.68905472755,69925.38329291344,4230.150585651398,9.16890573501587,0.0 +106500,0.4999995529651642,0.5470938086509705,,,,,,,,,,,,,, +107000,0.4999996423721313,0.6380360126495361,,,,,,,,,,,,,, +107500,0.4999996125698089,0.6940097808837891,,,,,,,,,,,,,, +108000,0.4999994933605194,0.6076784729957581,,,,,,,,,,,,,, +108500,0.4999996125698089,0.6440920829772949,,,,,,,,,,,,,, +109000,0.4999996125698089,0.6766055822372437,,,,,,,,,,,,,, +109305,,,0.8047472895408163,4.569540218431122,0.6078,4.565165,50000.0,0.5423,2.781318359375,10000.0,71917.54116845131,76446.3592095375,71917.54116845131,4339.878563404083,9.489476919174194,0.0 +109500,0.4999995529651642,0.5895081758499146,,,,,,,,,,,,,, +110000,0.4999995827674866,0.6342955827713013,,,,,,,,,,,,,, +110500,0.4999995231628418,0.6534225940704346,,,,,,,,,,,,,, +111000,0.4999995827674866,0.6540464162826538,,,,,,,,,,,,,, +111500,0.4999996125698089,0.5493713617324829,,,,,,,,,,,,,, +112000,0.4999995529651642,0.5500057935714722,,,,,,,,,,,,,, +112302,,,0.8119818239795918,4.524528036312181,0.60438,4.533105,50000.0,0.5399,2.811767578125,10000.0,73911.98550271988,78569.98849368095,73911.98550271988,4463.378042936325,9.768167972564695,0.0 +112500,0.4999995827674866,0.612583339214325,,,,,,,,,,,,,, +113000,0.4999996125698089,0.5453792810440063,,,,,,,,,,,,,, +113500,0.4999996423721313,0.661439061164856,,,,,,,,,,,,,, +114000,0.4999995827674866,0.6342620849609375,,,,,,,,,,,,,, +114500,0.4999995529651642,0.5982855558395386,,,,,,,,,,,,,, +115000,0.4999995529651642,0.6190052628517151,,,,,,,,,,,,,, +115412,,,,,,,,,,,75904.17921590805,,,,,0.0 diff --git a/logs/self_tuning/ademamix_golden/study_0/imagenet_resnet_pytorch/trial_1/meta_data_0.json b/logs/self_tuning/ademamix_golden/study_0/imagenet_resnet_pytorch/trial_1/meta_data_0.json new file mode 100644 index 00000000..6b6cdfc9 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/imagenet_resnet_pytorch/trial_1/meta_data_0.json @@ -0,0 +1,71 @@ +{ + "workload.bn_init_scale": 0.0, + "workload.center_crop_size": 224, + "workload.eval_batch_size": 1024, + "workload.eval_num_workers": 0, + "workload.eval_period_time_sec": 1996, + "workload.max_allowed_runtime_sec": 49918, + "workload.num_eval_train_examples": 50176, + "workload.num_test_examples": 10000, + "workload.num_train_examples": 1281167, + "workload.num_validation_examples": 50000, + "workload.resize_size": 256, + "workload.step_hint": 195999, + "workload.target_metric_name": "accuracy", + "workload.test_target_value": 0.656, + "workload.use_gelu": false, + "workload.use_silu": false, + "workload.validation_target_value": 0.77431, + "cpu.util.avg_percent_since_last": 6.2, + "cpu.freq.current": 2200.1639999999993, + "mem.total": 359053524992, + "mem.available": 347735846912, + "mem.used": 8161865728, + "mem.percent_used": 3.2, + "mem.read_bytes_since_boot": 18156542976, + "mem.write_bytes_since_boot": 40135066624, + "net.bytes_sent_since_boot": 31363294, + "net.bytes_recv_since_boot": 31366326, + "gpu.count": 4, + "gpu.0.compute.util": 0.0, + "gpu.0.mem.util": 0.0322998046875, + "gpu.0.mem.total": 40960.0, + "gpu.0.mem.used": 1323.0, + "gpu.0.mem.free": 39004.0, + "gpu.0.temp.current": 35.0, + "gpu.1.compute.util": 0.0, + "gpu.1.mem.util": 0.0322998046875, + "gpu.1.mem.total": 40960.0, + "gpu.1.mem.used": 1323.0, + "gpu.1.mem.free": 39004.0, + "gpu.1.temp.current": 40.0, + "gpu.2.compute.util": 0.0, + "gpu.2.mem.util": 0.0322998046875, + "gpu.2.mem.total": 40960.0, + "gpu.2.mem.used": 1323.0, + "gpu.2.mem.free": 39004.0, + "gpu.2.temp.current": 35.0, + "gpu.3.compute.util": 0.0, + "gpu.3.mem.util": 0.0322998046875, + "gpu.3.mem.total": 40960.0, + "gpu.3.mem.used": 1323.0, + "gpu.3.mem.free": 39004.0, + "gpu.3.temp.current": 39.0, + "gpu.avg.compute.util": 0.0, + "gpu.avg.mem.util": 0.0322998046875, + "gpu.avg.mem.total": 40960.0, + "gpu.avg.mem.used": 1323.0, + "gpu.avg.mem.free": 39004.0, + "gpu.avg.temp.current": 37.25, + "os_platform": "Linux-6.1.0-44-cloud-amd64-x86_64-with-glibc2.31", + "python_version": "3.11.10", + "python_compiler": "GCC 9.4.0", + "git_branch": "main", + "git_commit_hash": "b21be29be0a1573fb4f78f849aea019cdb520862", + "cpu_model_name": "Intel(R) Xeon(R) CPU @ 2.20GHz", + "cpu_count": 24, + "gpu_model_name": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_driver": "550.90.12", + "rng_seed": -1820589156 +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_0/imagenet_vit_pytorch/imagenet_vit_pytorch_09-12-2026-10-28-23.log b/logs/self_tuning/ademamix_golden/study_0/imagenet_vit_pytorch/imagenet_vit_pytorch_09-12-2026-10-28-23.log new file mode 100644 index 00000000..0a301278 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/imagenet_vit_pytorch/imagenet_vit_pytorch_09-12-2026-10-28-23.log @@ -0,0 +1,401 @@ +torchrun --redirects 1:0,2:0,3:0 --standalone --nnodes=1 --nproc_per_node=4 submission_runner.py --framework=pytorch --workload=imagenet_vit --submission_path=submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py --data_dir=/data/imagenet/pytorch --experiment_dir=/experiment_runs --experiment_name=submissions_a100/ademamix_golden/study_0 --overwrite=True --save_checkpoints=False --rng_seed=1076931326 --imagenet_v2_data_dir=/data/imagenet/pytorch --torch_compile=true --tuning_ruleset=self 2>&1 | tee -a /logs/imagenet_vit_pytorch_09-12-2026-10-28-23.log +W0912 10:28:34.515000 9 site-packages/torch/distributed/run.py:803] +W0912 10:28:34.515000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0912 10:28:34.515000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0912 10:28:34.515000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-12 10:28:39.477854: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-12 10:28:39.477854: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-12 10:28:39.477854: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-12 10:28:39.477855: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789208919.502464 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789208919.502452 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789208919.502452 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789208919.502452 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789208919.510509 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789208919.510509 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789208919.510527 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789208919.510557 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789208919.538103 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789208919.538101 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789208919.538101 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789208919.538132 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789208919.538132 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789208919.538134 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789208919.538135 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789208919.538136 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789208919.538137 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789208919.538136 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789208919.538139 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789208919.538120 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789208919.538141 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789208919.538144 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789208919.538146 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789208919.538148 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789208940.680833 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789208940.680896 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789208940.689157 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789208940.751391 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W912 10:29:16.984454937 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0912 10:29:17.051424 140150815601856 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/imagenet_vit_pytorch. +I0912 10:29:17.051425 139702990288064 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/imagenet_vit_pytorch. +I0912 10:29:17.051450 140539788526784 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/imagenet_vit_pytorch. +I0912 10:29:17.051458 140719470167232 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/imagenet_vit_pytorch. +I0912 10:29:17.075769 139702990288064 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/imagenet_vit_pytorch/trial_1. +I0912 10:29:17.400849 139702990288064 submission_runner.py:242] Initializing dataset. +I0912 10:29:51.002749 139702990288064 submission_runner.py:251] Initializing model. +I0912 10:29:51.379265 139702990288064 submission_runner.py:290] Performing `torch.compile`. +I0912 10:29:55.401881 139702990288064 submission_runner.py:294] Initializing optimizer. +I0912 10:29:55.402816 139702990288064 submission_runner.py:299] Initializing metrics bundle. +I0912 10:29:55.402976 139702990288064 submission_runner.py:321] Initializing checkpoint and logger. +I0912 10:29:55.404257 139702990288064 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_0/imagenet_vit_pytorch/trial_1/meta_data_0.json. +I0912 10:29:55.773739 139702990288064 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_0/imagenet_vit_pytorch/trial_1/flags_0.json. +I0912 10:29:55.813847 139702990288064 submission_runner.py:359] Starting training loop. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +[rank3]:W0912 10:30:27.663000 41 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank0]:W0912 10:30:27.688000 38 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank2]:W0912 10:30:27.691000 40 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank1]:W0912 10:30:28.787000 39 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +I0912 10:32:59.124210 139683602360064 logging_writer.py:48] [0] global_step=0, grad_norm=0.364471, loss=6.90776 +I0912 10:32:59.724234 139702990288064 submission.py:307] 0) loss = 6.908, grad_norm = 0.364 +I0912 10:33:00.457758 139702990288064 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 10:35:19.588684 139702990288064 spec.py:346] Evaluating on the validation split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 10:36:28.817891 139702990288064 spec.py:363] Evaluating on the test split. +I0912 10:36:30.206950 139702990288064 dataset_info.py:707] Load dataset info from /data/imagenet/pytorch/imagenet_v2/matched-frequency/3.0.0 +I0912 10:36:30.257853 139702990288064 reader.py:262] Creating a tf.data.Dataset reading 16 files located in folders: /data/imagenet/pytorch/imagenet_v2/matched-frequency/3.0.0. +I0912 10:36:30.601527 139702990288064 logging_logger.py:49] Constructing tf.data.Dataset imagenet_v2 for split test, from /data/imagenet/pytorch/imagenet_v2/matched-frequency/3.0.0 +[rank3]:[E912 11:03:33.487342849 ProcessGroupNCCL.cpp:683] [Rank 3] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=21, OpType=ALLREDUCE, NumelIn=1, NumelOut=1, Timeout(ms)=603000) ran for 603068 milliseconds before timing out. +[rank3]:[E912 11:03:33.583775420 ProcessGroupNCCL.cpp:2241] [PG ID 0 PG GUID 0(default_pg) Rank 3] failure detected by watchdog at work sequence id: 21 PG status: last enqueued work: 22, last completed work: 20 +[rank3]:[E912 11:03:33.583813569 ProcessGroupNCCL.cpp:730] Stack trace of the failed collective not found, potentially because FlightRecorder is disabled. You can enable it by setting TORCH_NCCL_TRACE_BUFFER_SIZE to a non-zero value. +[rank3]:[E912 11:03:33.583889900 ProcessGroupNCCL.cpp:2573] [PG ID 0 PG GUID 0(default_pg) Rank 3] First PG on this rank to signal dumping. +[rank3]:[E912 11:03:34.433634636 ProcessGroupNCCL.cpp:1858] [PG ID 0 PG GUID 0(default_pg) Rank 3] Received a dump signal due to a collective timeout from this local rank and we will try our best to dump the debug info. Last enqueued NCCL work: 22, last completed NCCL work: 20.This is most likely caused by incorrect usages of collectives, e.g., wrong sizes used across ranks, the order of collectives is not same for all ranks or the scheduled collective, for some reason, didn't run. Additionally, this can be caused by GIL deadlock or other reasons such as network errors or bugs in the communications library (e.g. NCCL), etc. +[rank2]:[E912 11:03:34.433718376 ProcessGroupNCCL.cpp:1794] [PG ID 0 PG GUID 0(default_pg) Rank 2] Observed flight recorder dump signal from another rank via TCPStore. +[rank0]:[E912 11:03:34.433743639 ProcessGroupNCCL.cpp:1794] [PG ID 0 PG GUID 0(default_pg) Rank 0] Observed flight recorder dump signal from another rank via TCPStore. +[rank2]:[E912 11:03:34.433932498 ProcessGroupNCCL.cpp:1858] [PG ID 0 PG GUID 0(default_pg) Rank 2] Received a dump signal due to a collective timeout from rank 3 and we will try our best to dump the debug info. Last enqueued NCCL work: 20, last completed NCCL work: 20.This is most likely caused by incorrect usages of collectives, e.g., wrong sizes used across ranks, the order of collectives is not same for all ranks or the scheduled collective, for some reason, didn't run. Additionally, this can be caused by GIL deadlock or other reasons such as network errors or bugs in the communications library (e.g. NCCL), etc. +[rank0]:[E912 11:03:34.433963105 ProcessGroupNCCL.cpp:1858] [PG ID 0 PG GUID 0(default_pg) Rank 0] Received a dump signal due to a collective timeout from rank 3 and we will try our best to dump the debug info. Last enqueued NCCL work: 20, last completed NCCL work: 20.This is most likely caused by incorrect usages of collectives, e.g., wrong sizes used across ranks, the order of collectives is not same for all ranks or the scheduled collective, for some reason, didn't run. Additionally, this can be caused by GIL deadlock or other reasons such as network errors or bugs in the communications library (e.g. NCCL), etc. +[rank1]:[E912 11:03:34.433995696 ProcessGroupNCCL.cpp:1794] [PG ID 0 PG GUID 0(default_pg) Rank 1] Observed flight recorder dump signal from another rank via TCPStore. +[rank1]:[E912 11:03:34.434262236 ProcessGroupNCCL.cpp:1858] [PG ID 0 PG GUID 0(default_pg) Rank 1] Received a dump signal due to a collective timeout from rank 3 and we will try our best to dump the debug info. Last enqueued NCCL work: 20, last completed NCCL work: 20.This is most likely caused by incorrect usages of collectives, e.g., wrong sizes used across ranks, the order of collectives is not same for all ranks or the scheduled collective, for some reason, didn't run. Additionally, this can be caused by GIL deadlock or other reasons such as network errors or bugs in the communications library (e.g. NCCL), etc. +[rank3]:[E912 11:03:34.475248765 ProcessGroupNCCL.cpp:1575] [PG ID 0 PG GUID 0(default_pg) Rank 3] ProcessGroupNCCL preparing to dump debug info. Include stack trace: 1 +[rank2]:[E912 11:03:45.636420048 ProcessGroupNCCL.cpp:1575] [PG ID 0 PG GUID 0(default_pg) Rank 2] ProcessGroupNCCL preparing to dump debug info. Include stack trace: 1 +[rank3]:[E912 11:04:34.111169834 ProcessGroupNCCL.cpp:744] [Rank 3] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. +[rank3]:[E912 11:04:34.111207777 ProcessGroupNCCL.cpp:758] [Rank 3] To avoid data inconsistency, we are taking the entire process down. +[rank3]:[E912 11:04:34.305264877 ProcessGroupNCCL.cpp:2057] [PG ID 0 PG GUID 0(default_pg) Rank 3] Process group watchdog thread terminated with exception: [Rank 3] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=21, OpType=ALLREDUCE, NumelIn=1, NumelOut=1, Timeout(ms)=603000) ran for 603068 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:686 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7ffaf6a11b80 in /usr/local/lib/python3.11/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7ffaf7997407 in /usr/local/lib/python3.11/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1691 (0x7ffaf799c011 in /usr/local/lib/python3.11/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xdf (0x7ffaf799d25f in /usr/local/lib/python3.11/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xd6df4 (0x7ffbcd915df4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x8609 (0x7ffbce3d9609 in /usr/lib/x86_64-linux-gnu/libpthread.so.0) +frame #6: clone + 0x43 (0x7ffbce1a4353 in /usr/lib/x86_64-linux-gnu/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 0 PG GUID 0(default_pg) Rank 3] Process group watchdog thread terminated with exception: [Rank 3] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=21, OpType=ALLREDUCE, NumelIn=1, NumelOut=1, Timeout(ms)=603000) ran for 603068 milliseconds before timing out. +Exception raised from checkTimeout at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:686 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7ffaf6a11b80 in /usr/local/lib/python3.11/site-packages/torch/lib/libc10.so) +frame #1: c10d::ProcessGroupNCCL::WorkNCCL::checkTimeout(std::optional > >) + 0x247 (0x7ffaf7997407 in /usr/local/lib/python3.11/site-packages/torch/lib/libtorch_cuda.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x1691 (0x7ffaf799c011 in /usr/local/lib/python3.11/site-packages/torch/lib/libtorch_cuda.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0xdf (0x7ffaf799d25f in /usr/local/lib/python3.11/site-packages/torch/lib/libtorch_cuda.so) +frame #4: + 0xd6df4 (0x7ffbcd915df4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #5: + 0x8609 (0x7ffbce3d9609 in /usr/lib/x86_64-linux-gnu/libpthread.so.0) +frame #6: clone + 0x43 (0x7ffbce1a4353 in /usr/lib/x86_64-linux-gnu/libc.so.6) + +Exception raised from run at /pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2063 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x80 (0x7ffaf6a11b80 in /usr/local/lib/python3.11/site-packages/torch/lib/libc10.so) +frame #1: + 0xe336d1 (0x7ffaf79736d1 in /usr/local/lib/python3.11/site-packages/torch/lib/libtorch_cuda.so) +frame #2: + 0x95044f (0x7ffaf749044f in /usr/local/lib/python3.11/site-packages/torch/lib/libtorch_cuda.so) +frame #3: + 0xd6df4 (0x7ffbcd915df4 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6) +frame #4: + 0x8609 (0x7ffbce3d9609 in /usr/lib/x86_64-linux-gnu/libpthread.so.0) +frame #5: clone + 0x43 (0x7ffbce1a4353 in /usr/lib/x86_64-linux-gnu/libc.so.6) + +Fatal Python error: Aborted + +Thread 0x00007ff832ffd700 (most recent call first): + + +Thread 0x00007ff8337fe700 (most recent call first): + + +Thread 0x00007ff833fff700 (most recent call first): + + +Thread 0x00007ff848ff9700 (most recent call first): + + +Thread 0x00007ff8497fa700 (most recent call first): + File "/usr/local/lib/python3.11/site-packages/torch/_inductor/compile_worker/subproc_pool.py", line 73 in _recv_msg + File "/usr/local/lib/python3.11/site-packages/torch/_inductor/compile_worker/subproc_pool.py", line 228 in _read_thread + File "/usr/local/lib/python3.11/threading.py", line 982 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ff849ffb700 (most recent call first): + File "/usr/local/lib/python3.11/threading.py", line 331 in wait + File "/usr/local/lib/python3.11/threading.py", line 629 in wait + File "/usr/local/lib/python3.11/site-packages/tqdm/_monitor.py", line 60 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ff84a7fc700 (most recent call first): + File "/usr/local/lib/python3.11/threading.py", line 327 in wait + File "/usr/local/lib/python3.11/multiprocessing/queues.py", line 231 in _feed + File "/usr/local/lib/python3.11/threading.py", line 982 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ff84affd700 (most recent call first): + File "/usr/local/lib/python3.11/threading.py", line 327 in wait + File "/usr/local/lib/python3.11/multiprocessing/queues.py", line 231 in _feed + File "/usr/local/lib/python3.11/threading.py", line 982 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ff84b7fe700 (most recent call first): + File "/usr/local/lib/python3.11/threading.py", line 327 in wait + File "/usr/local/lib/python3.11/multiprocessing/queues.py", line 231 in _feed + File "/usr/local/lib/python3.11/threading.py", line 982 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ff84bfff700 (most recent call first): + File "/usr/local/lib/python3.11/threading.py", line 327 in wait + File "/usr/local/lib/python3.11/multiprocessing/queues.py", line 231 in _feed + File "/usr/local/lib/python3.11/threading.py", line 982 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ff868ff9700 (most recent call first): + File "/usr/local/lib/python3.11/threading.py", line 327 in wait + File "/usr/local/lib/python3.11/multiprocessing/queues.py", line 231 in _feed + File "/usr/local/lib/python3.11/threading.py", line 982 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ff8697fa700 (most recent call first): + File "/usr/local/lib/python3.11/threading.py", line 327 in wait + File "/usr/local/lib/python3.11/multiprocessing/queues.py", line 231 in _feed + File "/usr/local/lib/python3.11/threading.py", line 982 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ff869ffb700 (most recent call first): + File "/usr/local/lib/python3.11/threading.py", line 327 in wait + File "/usr/local/lib/python3.11/multiprocessing/queues.py", line 231 in _feed + File "/usr/local/lib/python3.11/threading.py", line 982 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ff86a7fc700 (most recent call first): + File "/usr/local/lib/python3.11/threading.py", line 327 in wait + File "/usr/local/lib/python3.11/multiprocessing/queues.py", line 231 in _feed + File "/usr/local/lib/python3.11/threading.py", line 982 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ff86affd700 (most recent call first): + File "/usr/local/lib/python3.11/threading.py", line 327 in wait + File "/usr/local/lib/python3.11/multiprocessing/queues.py", line 231 in _feed + File "/usr/local/lib/python3.11/threading.py", line 982 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ff86b7fe700 (most recent call first): + File "/usr/local/lib/python3.11/threading.py", line 327 in wait + File "/usr/local/lib/python3.11/multiprocessing/queues.py", line 231 in _feed + File "/usr/local/lib/python3.11/threading.py", line 982 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ff86bfff700 (most recent call first): + File "/usr/local/lib/python3.11/threading.py", line 327 in wait + File "/usr/local/lib/python3.11/multiprocessing/queues.py", line 231 in _feed + File "/usr/local/lib/python3.11/threading.py", line 982 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ff888ff9700 (most recent call first): + File "/usr/local/lib/python3.11/threading.py", line 327 in wait + File "/usr/local/lib/python3.11/multiprocessing/queues.py", line 231 in _feed + File "/usr/local/lib/python3.11/threading.py", line 982 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ff8897fa700 (most recent call first): + File "/usr/local/lib/python3.11/threading.py", line 327 in wait + File "/usr/local/lib/python3.11/multiprocessing/queues.py", line 231 in _feed + File "/usr/local/lib/python3.11/threading.py", line 982 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ff889ffb700 (most recent call first): + File "/usr/local/lib/python3.11/threading.py", line 327 in wait + File "/usr/local/lib/python3.11/multiprocessing/queues.py", line 231 in _feed + File "/usr/local/lib/python3.11/threading.py", line 982 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ff88a7fc700 (most recent call first): + File "/usr/local/lib/python3.11/threading.py", line 327 in wait + File "/usr/local/lib/python3.11/multiprocessing/queues.py", line 231 in _feed + File "/usr/local/lib/python3.11/threading.py", line 982 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ff88affd700 (most recent call first): + File "/usr/local/lib/python3.11/threading.py", line 327 in wait + File "/usr/local/lib/python3.11/multiprocessing/queues.py", line 231 in _feed + File "/usr/local/lib/python3.11/threading.py", line 982 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ff88b7fe700 (most recent call first): + File "/usr/local/lib/python3.11/threading.py", line 327 in wait + File "/usr/local/lib/python3.11/multiprocessing/queues.py", line 231 in _feed + File "/usr/local/lib/python3.11/threading.py", line 982 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ff88bfff700 (most recent call first): + File "/usr/local/lib/python3.11/threading.py", line 327 in wait + File "/usr/local/lib/python3.11/multiprocessing/queues.py", line 231 in _feed + File "/usr/local/lib/python3.11/threading.py", line 982 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ff898ffd700 (most recent call first): + File "/usr/local/lib/python3.11/threading.py", line 327 in wait + File "/usr/local/lib/python3.11/multiprocessing/queues.py", line 231 in _feed + File "/usr/local/lib/python3.11/threading.py", line 982 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ff8997fe700 (most recent call first): + File "/usr/local/lib/python3.11/threading.py", line 327 in wait + File "/usr/local/lib/python3.11/multiprocessing/queues.py", line 231 in _feed + File "/usr/local/lib/python3.11/threading.py", line 982 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ff899fff700 (most recent call first): + File "/usr/local/lib/python3.11/selectors.py", line 415 in select + File "/usr/local/lib/python3.11/multiprocessing/connection.py", line 948 in wait + File "/usr/local/lib/python3.11/multiprocessing/connection.py", line 440 in _poll + File "/usr/local/lib/python3.11/multiprocessing/connection.py", line 257 in poll + File "/usr/local/lib/python3.11/multiprocessing/queues.py", line 113 in get + File "/usr/local/lib/python3.11/site-packages/torch/utils/data/_utils/pin_memory.py", line 28 in do_one_step + File "/usr/local/lib/python3.11/site-packages/torch/utils/data/_utils/pin_memory.py", line 52 in _pin_memory_loop + File "/usr/local/lib/python3.11/threading.py", line 982 in run + File "/usr/local/lib/python3.11/threading.py", line 1045 in _bootstrap_inner + File "/usr/local/lib/python3.11/threading.py", line 1002 in _bootstrap + +Thread 0x00007ffbce0844c0 (most recent call first): + File "/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py", line 2935 in all_reduce + File "/usr/local/lib/python3.11/site-packages/torch/distributed/c10d_logger.py", line 81 in wrapper + File "/algorithmic-efficiency/algoperf/pytorch_utils.py", line 65 in sync_ddp_time + File "/algorithmic-efficiency/submission_runner.py", line 201 in _get_time_ddp + File "/algorithmic-efficiency/submission_runner.py", line 491 in train_once + File "/algorithmic-efficiency/submission_runner.py", line 747 in score_submission_on_workload + File "/algorithmic-efficiency/submission_runner.py", line 838 in main + File "/usr/local/lib/python3.11/site-packages/absl/app.py", line 254 in _run_main + File "/usr/local/lib/python3.11/site-packages/absl/app.py", line 308 in run + File "/algorithmic-efficiency/submission_runner.py", line 873 in + +Extension modules: jaxlib.cpu_feature_guard, numpy._core._multiarray_umath, numpy.linalg._umath_linalg, numpy.random._common, numpy.random.bit_generator, numpy.random._bounded_integers, numpy.random._mt19937, numpy.random.mtrand, numpy.random._philox, numpy.random._pcg64, numpy.random._sfc64, numpy.random._generator, google._upb._message, charset_normalizer.md, charset_normalizer.cd, simplejson._speedups, requests.packages.charset_normalizer.md, requests.packages.chardet.md, requests.packages.charset_normalizer.cd, requests.packages.chardet.cd, h5py._errors, h5py.defs, h5py._objects, h5py.h5, h5py.utils, h5py.h5t, h5py.h5s, h5py.h5ac, h5py.h5p, h5py.h5r, h5py._proxy, h5py._conv, h5py.h5z, h5py.h5a, h5py.h5d, h5py.h5ds, h5py.h5g, h5py.h5i, h5py.h5o, h5py.h5f, h5py.h5fd, h5py.h5pl, h5py.h5l, h5py._selector, _cyutility, scipy._cyutility, scipy._lib._ccallback_c, scipy.sparse._sparsetools, _csparsetools, scipy.sparse._csparsetools, PIL._imaging, psutil._psutil_linux, psutil._psutil_posix, pyarrow.lib, pandas._libs._cyutility, pandas._libs.tslibs.ccalendar, pandas._libs.tslibs.np_datetime, pandas._libs.tslibs.dtypes, pandas._libs.tslibs.base, pandas._libs.tslibs.nattype, pandas._libs.tslibs.timezones, pandas._libs.properties, pandas._libs.tslibs.fields, pandas._libs.tslibs.timedeltas, pandas._libs.tslibs.tzconversion, pandas._libs.tslibs.timestamps, pandas._libs.tslibs.offsets, pandas._libs.tslibs.strptime, pandas._libs.tslibs.parsing, pandas._libs.tslibs.conversion, pandas._libs.tslibs.period, pandas._libs.tslibs.vectorized, pandas._libs.ops_dispatch, pandas._libs.missing, pandas._libs.hashtable, pandas._libs.algos, pandas._libs.interval, pandas._libs.lib, pyarrow._compute, pandas._libs.ops, pandas._libs.hashing, pandas._libs.arrays, pandas._libs.tslib, pandas._libs.sparse, pandas._libs.internals, pandas._libs.indexing, pandas._libs.index, pandas._libs.writers, pandas._libs.join, pandas._libs.window.aggregations, pandas._libs.window.indexers, pandas._libs.reshape, pandas._libs.groupby, pandas._libs.json, pandas._libs.parsers, pandas._libs.testing, kiwisolver._cext, sklearn.__check_build._check_build, scipy.special._ufuncs_cxx, scipy.special._ellip_harm_2, scipy.special._special_ufuncs, scipy.special._gufuncs, scipy.special._ufuncs, scipy.special._specfun, scipy.special._comb, scipy.linalg._fblas, scipy.linalg._flapack, scipy.linalg.cython_lapack, scipy.linalg._cythonized_array_utils, scipy.linalg._solve_toeplitz, scipy.linalg._batched_linalg, scipy.linalg._decomp_lu_cython, scipy.linalg._matfuncs_schur_sqrtm, scipy.linalg._matfuncs_expm, scipy.linalg._linalg_pythran, scipy.linalg.cython_blas, scipy.linalg._decomp_update, scipy.sparse.linalg._dsolve._superlu, scipy.sparse.linalg._eigen.arpack._arpacklib, scipy.sparse.linalg._propack, scipy.spatial._ckdtree, scipy._lib.messagestream, scipy.spatial._qhull, scipy.spatial._voronoi, scipy.spatial._hausdorff, scipy.spatial._distance_wrap, scipy.spatial.transform._rotation_cy, scipy.spatial.transform._rigid_transform_cy, scipy.optimize._group_columns, scipy.optimize._trlib._trlib, scipy.optimize._lbfgsb, _moduleTNC, scipy.optimize._moduleTNC, scipy.optimize._slsqplib, scipy.optimize._minpack, scipy.optimize._lsq.givens_elimination, scipy.optimize._zeros, scipy._lib._uarray._uarray, scipy.linalg._decomp_interpolative, scipy.optimize._bglu_dense, scipy.optimize._lsap, scipy.optimize._direct, scipy.integrate._odepack, scipy.integrate._quadpack, scipy.integrate._vode, scipy.integrate._dop, scipy.interpolate._fitpack, scipy.interpolate._dfitpack, scipy.interpolate._dierckx, scipy.interpolate._ppoly, scipy.interpolate._interpnd, scipy.interpolate._rbfinterp_pythran, scipy.interpolate._rgi_cython, scipy.special.cython_special, scipy.stats._stats, scipy.stats._biasedurn, scipy.stats._stats_pythran, scipy.stats._levy_stable.levyst, scipy.stats._ansari_swilk_statistics, scipy.sparse.csgraph._tools, scipy.sparse.csgraph._shortest_path, scipy.sparse.csgraph._traversal, scipy.sparse.csgraph._min_spanning_tree, scipy.sparse.csgraph._flow, scipy.sparse.csgraph._matching, scipy.sparse.csgraph._reordering, scipy.stats._sobol, scipy.stats._qmc_cy, scipy.stats._rcont.rcont, scipy.stats._qmvnt_cy, scipy.ndimage._nd_image, scipy.ndimage._rank_filter_1d, _ni_label, scipy.ndimage._ni_label, sklearn.utils._isfinite, sklearn.utils.sparsefuncs_fast, sklearn.utils.murmurhash, sklearn.utils._openmp_helpers, torch._C, torch._C._dynamo.autograd_compiler, torch._C._dynamo.eval_frame, torch._C._dynamo.guards, torch._C._dynamo.utils, torch._C._fft, torch._C._linalg, torch._C._nested, torch._C._nn, torch._C._sparse, torch._C._special, uvloop.loop, msgpack._cmsgpack, yaml._yaml, PIL._imagingft, markupsafe._speedups, cuda_utils (total: 195) +[rank1]:[E912 11:06:16.415267582 ProcessGroupNCCL.cpp:1575] [PG ID 0 PG GUID 0(default_pg) Rank 1] ProcessGroupNCCL preparing to dump debug info. Include stack trace: 1 +[rank0]:[E912 11:06:30.628395674 ProcessGroupNCCL.cpp:1575] [PG ID 0 PG GUID 0(default_pg) Rank 0] ProcessGroupNCCL preparing to dump debug info. Include stack trace: 1 +W0912 11:07:01.517000 9 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 38 closing signal SIGTERM +W0912 11:07:01.525000 9 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 39 closing signal SIGTERM +W0912 11:07:01.653000 9 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 40 closing signal SIGTERM +E0912 11:07:01.721000 9 site-packages/torch/distributed/elastic/multiprocessing/api.py:882] failed (exitcode: -6) local_rank: 3 (pid: 41) of binary: /usr/local/bin/python3.11 +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 6, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/torch/distributed/run.py", line 936, in main + run(args) + File "/usr/local/lib/python3.11/site-packages/torch/distributed/run.py", line 927, in run + elastic_launch( + File "/usr/local/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 156, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 293, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +=================================================== +submission_runner.py FAILED +--------------------------------------------------- +Failures: + +--------------------------------------------------- +Root Cause (first observed failure): +[0]: + time : 2026-09-12_11:07:01 + host : df1b65fa5772 + rank : 3 (local_rank: 3) + exitcode : -6 (pid: 41) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 41 +=================================================== diff --git a/logs/self_tuning/ademamix_golden/study_0/imagenet_vit_pytorch/trial_1/events.out.tfevents.1789208995.df1b65fa5772.38.0.v2 b/logs/self_tuning/ademamix_golden/study_0/imagenet_vit_pytorch/trial_1/events.out.tfevents.1789208995.df1b65fa5772.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..fa1a215dca17f78d60635caae139a6312174d3e0 GIT binary patch literal 270 zcmcCxfPkBq9G-JBFdWG6T6xP+iZ`h!F*8rkwJbHS#L6g0g-fXW6 FlL68jPK*Em literal 0 HcmV?d00001 diff --git a/logs/self_tuning/ademamix_golden/study_0/imagenet_vit_pytorch/trial_1/flags_0.json b/logs/self_tuning/ademamix_golden/study_0/imagenet_vit_pytorch/trial_1/flags_0.json new file mode 100644 index 00000000..40c78f09 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/imagenet_vit_pytorch/trial_1/flags_0.json @@ -0,0 +1,63 @@ +{ + "logtostderr": false, + "alsologtostderr": false, + "log_dir": "", + "v": 0, + "verbosity": 0, + "logger_levels": {}, + "stderrthreshold": "fatal", + "showprefixforinfo": true, + "run_with_pdb": false, + "pdb_post_mortem": false, + "pdb": false, + "run_with_profiling": false, + "profile_file": null, + "use_cprofile_for_profiling": true, + "only_check_args": false, + "op_conversion_fallback_to_while_loop": true, + "delta_threshold": 0.5, + "tt_check_filter": false, + "tt_single_core_summaries": false, + "runtime_oom_exit": true, + "hbm_oom_exit": true, + "test_srcdir": "", + "test_tmpdir": "/tmp/absl_testing", + "test_random_seed": 301, + "test_randomize_ordering_seed": "", + "xml_output_file": "", + "experimental_orbax_use_distributed_process_id": false, + "experimental_orbax_use_distributed_barrier": false, + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "workload": "imagenet_vit", + "tuning_ruleset": "self", + "tuning_search_space": null, + "num_tuning_trials": 1, + "data_dir": "/data/imagenet/pytorch", + "imagenet_v2_data_dir": "/data/imagenet/pytorch", + "librispeech_tokenizer_vocab_path": "", + "framework": "pytorch", + "torch_compile": true, + "experiment_dir": "/experiment_runs", + "experiment_name": "submissions_a100/ademamix_golden/study_0", + "save_checkpoints": false, + "save_intermediate_checkpoints": true, + "resume_last_run": null, + "append_timestamp": false, + "use_wandb": false, + "profile": false, + "max_global_steps": null, + "overwrite": true, + "hparam_start_index": null, + "hparam_end_index": null, + "rng_seed": 1076931326, + "set_pytorch_max_split_size": false, + "pytorch_eval_num_workers": 0, + "capture_jax_trace": false, + "skip_evals": false, + "?": false, + "help": false, + "helpshort": false, + "helpfull": false, + "helpxml": false, + "tfds_debug_list_dir": false +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_0/imagenet_vit_pytorch/trial_1/measurements.csv b/logs/self_tuning/ademamix_golden/study_0/imagenet_vit_pytorch/trial_1/measurements.csv new file mode 100644 index 00000000..bc7b51a0 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/imagenet_vit_pytorch/trial_1/measurements.csv @@ -0,0 +1,2 @@ +global_step,grad_norm,loss +0,0.3644710183143616,6.907755374908447 diff --git a/logs/self_tuning/ademamix_golden/study_0/imagenet_vit_pytorch/trial_1/meta_data_0.json b/logs/self_tuning/ademamix_golden/study_0/imagenet_vit_pytorch/trial_1/meta_data_0.json new file mode 100644 index 00000000..346b12a1 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/imagenet_vit_pytorch/trial_1/meta_data_0.json @@ -0,0 +1,74 @@ +{ + "workload.bn_init_scale": 0.0, + "workload.center_crop_size": 224, + "workload.eval_batch_size": 2048, + "workload.eval_num_workers": 0, + "workload.eval_period_time_sec": 2571, + "workload.max_allowed_runtime_sec": 64292, + "workload.num_eval_train_examples": 51200, + "workload.num_test_examples": 10000, + "workload.num_train_examples": 1281167, + "workload.num_validation_examples": 50000, + "workload.resize_size": 256, + "workload.step_hint": 167999, + "workload.target_metric_name": "accuracy", + "workload.test_target_value": 0.6518999999999999, + "workload.use_gelu": false, + "workload.use_glu": false, + "workload.use_map": false, + "workload.use_post_layer_norm": false, + "workload.use_silu": false, + "workload.validation_target_value": 0.77309, + "cpu.util.avg_percent_since_last": 4.3, + "cpu.freq.current": 2200.1959999999985, + "mem.total": 359053524992, + "mem.available": 347782422528, + "mem.used": 8052224000, + "mem.percent_used": 3.1, + "mem.read_bytes_since_boot": 19036437884416, + "mem.write_bytes_since_boot": 47525721600, + "net.bytes_sent_since_boot": 32076517, + "net.bytes_recv_since_boot": 32079610, + "gpu.count": 4, + "gpu.0.compute.util": 0.0, + "gpu.0.mem.util": 0.0314208984375, + "gpu.0.mem.total": 40960.0, + "gpu.0.mem.used": 1287.0, + "gpu.0.mem.free": 39040.0, + "gpu.0.temp.current": 35.0, + "gpu.1.compute.util": 0.0, + "gpu.1.mem.util": 0.0314208984375, + "gpu.1.mem.total": 40960.0, + "gpu.1.mem.used": 1287.0, + "gpu.1.mem.free": 39040.0, + "gpu.1.temp.current": 31.0, + "gpu.2.compute.util": 0.0, + "gpu.2.mem.util": 0.0314208984375, + "gpu.2.mem.total": 40960.0, + "gpu.2.mem.used": 1287.0, + "gpu.2.mem.free": 39040.0, + "gpu.2.temp.current": 32.0, + "gpu.3.compute.util": 0.0, + "gpu.3.mem.util": 0.0314208984375, + "gpu.3.mem.total": 40960.0, + "gpu.3.mem.used": 1287.0, + "gpu.3.mem.free": 39040.0, + "gpu.3.temp.current": 34.0, + "gpu.avg.compute.util": 0.0, + "gpu.avg.mem.util": 0.0314208984375, + "gpu.avg.mem.total": 40960.0, + "gpu.avg.mem.used": 1287.0, + "gpu.avg.mem.free": 39040.0, + "gpu.avg.temp.current": 33.0, + "os_platform": "Linux-6.1.0-44-cloud-amd64-x86_64-with-glibc2.31", + "python_version": "3.11.10", + "python_compiler": "GCC 9.4.0", + "git_branch": "main", + "git_commit_hash": "b21be29be0a1573fb4f78f849aea019cdb520862", + "cpu_model_name": "Intel(R) Xeon(R) CPU @ 2.20GHz", + "cpu_count": 24, + "gpu_model_name": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_driver": "550.90.12", + "rng_seed": 1076931326 +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_0/librispeech_conformer_pytorch/librispeech_conformer_pytorch_09-15-2026-04-25-35.log b/logs/self_tuning/ademamix_golden/study_0/librispeech_conformer_pytorch/librispeech_conformer_pytorch_09-15-2026-04-25-35.log new file mode 100644 index 00000000..bd62abce --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/librispeech_conformer_pytorch/librispeech_conformer_pytorch_09-15-2026-04-25-35.log @@ -0,0 +1,1011 @@ +torchrun --redirects 1:0,2:0,3:0 --standalone --nnodes=1 --nproc_per_node=4 submission_runner.py --framework=pytorch --workload=librispeech_conformer --submission_path=submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py --data_dir=/data/librispeech --experiment_dir=/experiment_runs --experiment_name=submissions_a100/ademamix_golden/study_0 --overwrite=True --save_checkpoints=False --rng_seed=1128445477 --librispeech_tokenizer_vocab_path=/data/librispeech/spm_model.vocab --torch_compile=true --tuning_ruleset=self 2>&1 | tee -a /logs/librispeech_conformer_pytorch_09-15-2026-04-25-35.log +W0915 04:26:01.267000 9 site-packages/torch/distributed/run.py:803] +W0915 04:26:01.267000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0915 04:26:01.267000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0915 04:26:01.267000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-15 04:26:15.633485: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-15 04:26:15.633485: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-15 04:26:15.633485: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-15 04:26:15.633484: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789446376.212856 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789446376.212858 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789446376.212860 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789446376.212883 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789446376.324055 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789446376.324055 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789446376.324055 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789446376.324071 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789446377.713451 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789446377.713449 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789446377.713451 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789446377.713452 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789446377.713493 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789446377.713493 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789446377.713493 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789446377.713496 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789446377.713496 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789446377.713496 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789446377.713496 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789446377.713498 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789446377.713499 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789446377.713499 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789446377.713499 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789446377.713502 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789446404.748177 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789446404.748201 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789446404.866749 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789446404.880519 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W915 04:26:54.662417492 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0915 04:26:56.936811 140016250610880 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/librispeech_conformer_pytorch. +I0915 04:26:56.936805 139885556876480 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/librispeech_conformer_pytorch. +I0915 04:26:56.936809 140198472086720 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/librispeech_conformer_pytorch. +I0915 04:26:56.936816 139627431158976 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/librispeech_conformer_pytorch. +I0915 04:26:57.213325 139627431158976 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/librispeech_conformer_pytorch/trial_1. +I0915 04:26:57.489902 139627431158976 submission_runner.py:242] Initializing dataset. +I0915 04:26:57.490084 139627431158976 input_pipeline.py:19] Loading split = train-clean-100 +I0915 04:26:57.554991 139627431158976 input_pipeline.py:19] Loading split = train-clean-360 +I0915 04:26:57.723695 139627431158976 input_pipeline.py:19] Loading split = train-other-500 +I0915 04:26:58.214523 139627431158976 submission_runner.py:251] Initializing model. +W0915 04:27:07.663937 140016250610880 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0915 04:27:07.663936 139627431158976 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0915 04:27:07.663930 140198472086720 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0915 04:27:07.663984 139885556876480 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +I0915 04:27:07.664132 139627431158976 submission_runner.py:294] Initializing optimizer. +I0915 04:27:07.664843 139627431158976 submission_runner.py:299] Initializing metrics bundle. +I0915 04:27:07.664999 139627431158976 submission_runner.py:321] Initializing checkpoint and logger. +I0915 04:27:07.667343 139627431158976 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_0/librispeech_conformer_pytorch/trial_1/meta_data_0.json. +I0915 04:27:07.667461 140016250610880 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0915 04:27:07.667497 139885556876480 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0915 04:27:07.667516 140198472086720 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0915 04:27:07.667578 140016250610880 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0915 04:27:07.667575 139627431158976 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0915 04:27:07.667632 139627431158976 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0915 04:27:07.667627 139885556876480 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0915 04:27:07.667645 140198472086720 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0915 04:27:08.053236 139627431158976 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_0/librispeech_conformer_pytorch/trial_1/flags_0.json. +I0915 04:27:08.074439 139627431158976 submission_runner.py:359] Starting training loop. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/algorithmic-efficiency/algoperf/workloads/librispeech_conformer/librispeech_pytorch/preprocessor.py:516: UserWarning: Specified kernel cache directory could not be created! This disables kernel caching. Specified directory is /root/.cache/torch/kernels. This warning will appear only once per process. (Triggered internally at /pytorch/aten/src/ATen/native/cuda/jit_utils.cpp:1487.) + spectrum = torch.abs(spectrum) +/algorithmic-efficiency/algoperf/workloads/librispeech_conformer/librispeech_pytorch/preprocessor.py:516: UserWarning: Specified kernel cache directory could not be created! This disables kernel caching. Specified directory is /root/.cache/torch/kernels. This warning will appear only once per process. (Triggered internally at /pytorch/aten/src/ATen/native/cuda/jit_utils.cpp:1487.) + spectrum = torch.abs(spectrum) +/algorithmic-efficiency/algoperf/workloads/librispeech_conformer/librispeech_pytorch/preprocessor.py:516: UserWarning: Specified kernel cache directory could not be created! This disables kernel caching. Specified directory is /root/.cache/torch/kernels. This warning will appear only once per process. (Triggered internally at /pytorch/aten/src/ATen/native/cuda/jit_utils.cpp:1487.) + spectrum = torch.abs(spectrum) +I0915 04:27:32.076073 139610205947648 logging_writer.py:48] [0] global_step=0, grad_norm=0.5, loss=33.2167 +I0915 04:27:32.305764 139627431158976 submission.py:307] 0) loss = 33.217, grad_norm = 0.500 +I0915 04:27:33.313863 139627431158976 spec.py:333] Evaluating on the training split. +I0915 04:27:33.315290 139627431158976 input_pipeline.py:19] Loading split = train-clean-100 +I0915 04:27:33.351150 139627431158976 input_pipeline.py:19] Loading split = train-clean-360 +I0915 04:27:33.477039 139627431158976 input_pipeline.py:19] Loading split = train-other-500 +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 04:28:07.709419 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 04:28:07.710832 139627431158976 input_pipeline.py:19] Loading split = dev-clean +I0915 04:28:07.714587 139627431158976 input_pipeline.py:19] Loading split = dev-other +I0915 04:28:28.084739 139627431158976 spec.py:363] Evaluating on the test split. +I0915 04:28:28.086111 139627431158976 input_pipeline.py:19] Loading split = test-clean +I0915 04:28:38.542618 139627431158976 submission_runner.py:516] Time since start: 90.47s, Step: 1, {'train/ctc_loss': 31.778551183445963, 'train/wer': 1.6197443941103737, 'validation/ctc_loss': 30.278628566405466, 'validation/wer': 1.2183266547578815, 'validation/num_examples': 5348, 'test/ctc_loss': 30.430827795341635, 'test/wer': 1.2926695509109742, 'test/num_examples': 2472, 'score': 24.232906341552734, 'total_duration': 90.46797966957092, 'accumulated_submission_time': 24.232906341552734, 'accumulated_eval_time': 65.22912073135376, 'accumulated_logging_time': 0} +I0915 04:28:38.586603 139606351599360 logging_writer.py:48] [1] accumulated_eval_time=65.2291, accumulated_logging_time=0, accumulated_submission_time=24.2329, global_step=1, preemption_count=0, score=24.2329, test/ctc_loss=30.4308, test/num_examples=2472, test/wer=1.29267, total_duration=90.468, train/ctc_loss=31.7786, train/wer=1.61974, validation/ctc_loss=30.2786, validation/num_examples=5348, validation/wer=1.21833 +I0915 04:28:41.206759 139606343206656 logging_writer.py:48] [1] global_step=1, grad_norm=0.5, loss=32.6755 +I0915 04:28:41.210014 139627431158976 submission.py:307] 1) loss = 32.675, grad_norm = 0.500 +I0915 04:28:41.974556 139606351599360 logging_writer.py:48] [2] global_step=2, grad_norm=0.5, loss=33.1776 +I0915 04:28:41.978602 139627431158976 submission.py:307] 2) loss = 33.178, grad_norm = 0.500 +I0915 04:28:42.437041 139606343206656 logging_writer.py:48] [3] global_step=3, grad_norm=0.5, loss=33.2417 +I0915 04:28:42.441176 139627431158976 submission.py:307] 3) loss = 33.242, grad_norm = 0.500 +I0915 04:28:42.888185 139606351599360 logging_writer.py:48] [4] global_step=4, grad_norm=0.5, loss=32.7842 +I0915 04:28:42.891241 139627431158976 submission.py:307] 4) loss = 32.784, grad_norm = 0.500 +I0915 04:28:43.342317 139606343206656 logging_writer.py:48] [5] global_step=5, grad_norm=0.5, loss=32.9411 +I0915 04:28:43.345634 139627431158976 submission.py:307] 5) loss = 32.941, grad_norm = 0.500 +I0915 04:28:43.806505 139606351599360 logging_writer.py:48] [6] global_step=6, grad_norm=0.5, loss=32.995 +I0915 04:28:43.809547 139627431158976 submission.py:307] 6) loss = 32.995, grad_norm = 0.500 +I0915 04:28:44.260364 139606343206656 logging_writer.py:48] [7] global_step=7, grad_norm=0.5, loss=32.0085 +I0915 04:28:44.263767 139627431158976 submission.py:307] 7) loss = 32.009, grad_norm = 0.500 +I0915 04:28:44.726717 139606351599360 logging_writer.py:48] [8] global_step=8, grad_norm=0.5, loss=32.4013 +I0915 04:28:44.731092 139627431158976 submission.py:307] 8) loss = 32.401, grad_norm = 0.500 +I0915 04:28:45.255091 139606343206656 logging_writer.py:48] [9] global_step=9, grad_norm=0.5, loss=31.9509 +I0915 04:28:45.266106 139627431158976 submission.py:307] 9) loss = 31.951, grad_norm = 0.500 +I0915 04:28:46.939193 139606351599360 logging_writer.py:48] [10] global_step=10, grad_norm=0.5, loss=31.7816 +I0915 04:28:46.942273 139627431158976 submission.py:307] 10) loss = 31.782, grad_norm = 0.500 +I0915 04:28:47.576637 139606343206656 logging_writer.py:48] [11] global_step=11, grad_norm=0.5, loss=31.8955 +I0915 04:28:47.579805 139627431158976 submission.py:307] 11) loss = 31.896, grad_norm = 0.500 +I0915 04:28:48.439808 139606351599360 logging_writer.py:48] [12] global_step=12, grad_norm=0.5, loss=31.9211 +I0915 04:28:48.444005 139627431158976 submission.py:307] 12) loss = 31.921, grad_norm = 0.500 +I0915 04:28:50.086056 139606343206656 logging_writer.py:48] [13] global_step=13, grad_norm=0.5, loss=31.1502 +I0915 04:28:50.091142 139627431158976 submission.py:307] 13) loss = 31.150, grad_norm = 0.500 +I0915 04:28:52.046370 139606351599360 logging_writer.py:48] [14] global_step=14, grad_norm=0.5, loss=30.5811 +I0915 04:28:52.051340 139627431158976 submission.py:307] 14) loss = 30.581, grad_norm = 0.500 +I0915 04:28:53.009060 139606343206656 logging_writer.py:48] [15] global_step=15, grad_norm=0.5, loss=29.8363 +I0915 04:28:53.013272 139627431158976 submission.py:307] 15) loss = 29.836, grad_norm = 0.500 +I0915 04:28:53.789121 139606351599360 logging_writer.py:48] [16] global_step=16, grad_norm=0.5, loss=29.7874 +I0915 04:28:53.793604 139627431158976 submission.py:307] 16) loss = 29.787, grad_norm = 0.500 +I0915 04:28:55.435497 139606343206656 logging_writer.py:48] [17] global_step=17, grad_norm=0.5, loss=29.0253 +I0915 04:28:55.438582 139627431158976 submission.py:307] 17) loss = 29.025, grad_norm = 0.500 +I0915 04:28:57.039916 139606351599360 logging_writer.py:48] [18] global_step=18, grad_norm=0.5, loss=28.1014 +I0915 04:28:57.043015 139627431158976 submission.py:307] 18) loss = 28.101, grad_norm = 0.500 +I0915 04:28:58.535443 139606343206656 logging_writer.py:48] [19] global_step=19, grad_norm=0.5, loss=26.6897 +I0915 04:28:58.539139 139627431158976 submission.py:307] 19) loss = 26.690, grad_norm = 0.500 +I0915 04:28:58.991389 139606351599360 logging_writer.py:48] [20] global_step=20, grad_norm=0.5, loss=25.3466 +I0915 04:28:58.994471 139627431158976 submission.py:307] 20) loss = 25.347, grad_norm = 0.500 +I0915 04:29:00.731663 139606343206656 logging_writer.py:48] [21] global_step=21, grad_norm=0.5, loss=23.9502 +I0915 04:29:00.735356 139627431158976 submission.py:307] 21) loss = 23.950, grad_norm = 0.500 +I0915 04:29:02.218164 139606351599360 logging_writer.py:48] [22] global_step=22, grad_norm=0.5, loss=22.7294 +I0915 04:29:02.221246 139627431158976 submission.py:307] 22) loss = 22.729, grad_norm = 0.500 +I0915 04:29:04.011965 139606343206656 logging_writer.py:48] [23] global_step=23, grad_norm=0.5, loss=20.5213 +I0915 04:29:04.015057 139627431158976 submission.py:307] 23) loss = 20.521, grad_norm = 0.500 +I0915 04:29:04.467544 139606351599360 logging_writer.py:48] [24] global_step=24, grad_norm=0.5, loss=18.9489 +I0915 04:29:04.470767 139627431158976 submission.py:307] 24) loss = 18.949, grad_norm = 0.500 +I0915 04:29:06.389887 139606343206656 logging_writer.py:48] [25] global_step=25, grad_norm=0.5, loss=17.0183 +I0915 04:29:06.392984 139627431158976 submission.py:307] 25) loss = 17.018, grad_norm = 0.500 +I0915 04:29:07.827448 139606351599360 logging_writer.py:48] [26] global_step=26, grad_norm=0.5, loss=15.1488 +I0915 04:29:07.830569 139627431158976 submission.py:307] 26) loss = 15.149, grad_norm = 0.500 +I0915 04:29:09.687259 139606343206656 logging_writer.py:48] [27] global_step=27, grad_norm=0.5, loss=13.3485 +I0915 04:29:09.691061 139627431158976 submission.py:307] 27) loss = 13.348, grad_norm = 0.500 +I0915 04:29:10.150123 139606351599360 logging_writer.py:48] [28] global_step=28, grad_norm=0.5, loss=11.6607 +I0915 04:29:10.154244 139627431158976 submission.py:307] 28) loss = 11.661, grad_norm = 0.500 +I0915 04:29:11.285987 139606343206656 logging_writer.py:48] [29] global_step=29, grad_norm=0.5, loss=10.1033 +I0915 04:29:11.289313 139627431158976 submission.py:307] 29) loss = 10.103, grad_norm = 0.500 +I0915 04:29:12.688395 139606351599360 logging_writer.py:48] [30] global_step=30, grad_norm=0.5, loss=8.98001 +I0915 04:29:12.693555 139627431158976 submission.py:307] 30) loss = 8.980, grad_norm = 0.500 +I0915 04:29:14.944298 139606343206656 logging_writer.py:48] [31] global_step=31, grad_norm=0.5, loss=8.15704 +I0915 04:29:14.947376 139627431158976 submission.py:307] 31) loss = 8.157, grad_norm = 0.500 +I0915 04:29:15.393041 139606351599360 logging_writer.py:48] [32] global_step=32, grad_norm=0.5, loss=7.57199 +I0915 04:29:15.396205 139627431158976 submission.py:307] 32) loss = 7.572, grad_norm = 0.500 +I0915 04:29:16.646088 139606343206656 logging_writer.py:48] [33] global_step=33, grad_norm=0.5, loss=7.32775 +I0915 04:29:16.649191 139627431158976 submission.py:307] 33) loss = 7.328, grad_norm = 0.500 +I0915 04:29:18.333241 139606351599360 logging_writer.py:48] [34] global_step=34, grad_norm=0.5, loss=7.25233 +I0915 04:29:18.336422 139627431158976 submission.py:307] 34) loss = 7.252, grad_norm = 0.500 +I0915 04:29:20.283773 139606343206656 logging_writer.py:48] [35] global_step=35, grad_norm=0.5, loss=7.28429 +I0915 04:29:20.286879 139627431158976 submission.py:307] 35) loss = 7.284, grad_norm = 0.500 +I0915 04:29:20.731502 139606351599360 logging_writer.py:48] [36] global_step=36, grad_norm=0.5, loss=7.30559 +I0915 04:29:20.734534 139627431158976 submission.py:307] 36) loss = 7.306, grad_norm = 0.500 +I0915 04:29:21.829573 139606343206656 logging_writer.py:48] [37] global_step=37, grad_norm=0.5, loss=7.29651 +I0915 04:29:21.832803 139627431158976 submission.py:307] 37) loss = 7.297, grad_norm = 0.500 +I0915 04:29:23.362182 139606351599360 logging_writer.py:48] [38] global_step=38, grad_norm=0.5, loss=7.22494 +I0915 04:29:23.365495 139627431158976 submission.py:307] 38) loss = 7.225, grad_norm = 0.500 +I0915 04:29:25.592481 139606343206656 logging_writer.py:48] [39] global_step=39, grad_norm=0.5, loss=7.16031 +I0915 04:29:25.595558 139627431158976 submission.py:307] 39) loss = 7.160, grad_norm = 0.500 +I0915 04:29:26.041002 139606351599360 logging_writer.py:48] [40] global_step=40, grad_norm=0.5, loss=7.08573 +I0915 04:29:26.044222 139627431158976 submission.py:307] 40) loss = 7.086, grad_norm = 0.500 +I0915 04:29:26.851862 139606343206656 logging_writer.py:48] [41] global_step=41, grad_norm=0.5, loss=7.05783 +I0915 04:29:26.855261 139627431158976 submission.py:307] 41) loss = 7.058, grad_norm = 0.500 +I0915 04:29:28.993901 139606351599360 logging_writer.py:48] [42] global_step=42, grad_norm=0.5, loss=7.05228 +I0915 04:29:28.996969 139627431158976 submission.py:307] 42) loss = 7.052, grad_norm = 0.500 +I0915 04:29:30.825048 139606343206656 logging_writer.py:48] [43] global_step=43, grad_norm=0.5, loss=7.01941 +I0915 04:29:30.828185 139627431158976 submission.py:307] 43) loss = 7.019, grad_norm = 0.500 +I0915 04:29:31.272846 139606351599360 logging_writer.py:48] [44] global_step=44, grad_norm=0.5, loss=6.97617 +I0915 04:29:31.275983 139627431158976 submission.py:307] 44) loss = 6.976, grad_norm = 0.500 +I0915 04:29:32.127108 139606343206656 logging_writer.py:48] [45] global_step=45, grad_norm=0.5, loss=6.91257 +I0915 04:29:32.130324 139627431158976 submission.py:307] 45) loss = 6.913, grad_norm = 0.500 +I0915 04:29:34.202074 139606351599360 logging_writer.py:48] [46] global_step=46, grad_norm=0.5, loss=6.8757 +I0915 04:29:34.205204 139627431158976 submission.py:307] 46) loss = 6.876, grad_norm = 0.500 +I0915 04:29:35.817160 139606343206656 logging_writer.py:48] [47] global_step=47, grad_norm=0.5, loss=6.84831 +I0915 04:29:35.820397 139627431158976 submission.py:307] 47) loss = 6.848, grad_norm = 0.500 +I0915 04:29:36.369773 139606351599360 logging_writer.py:48] [48] global_step=48, grad_norm=0.5, loss=6.80805 +I0915 04:29:36.372924 139627431158976 submission.py:307] 48) loss = 6.808, grad_norm = 0.500 +I0915 04:29:37.453343 139606343206656 logging_writer.py:48] [49] global_step=49, grad_norm=0.5, loss=6.74558 +I0915 04:29:37.456583 139627431158976 submission.py:307] 49) loss = 6.746, grad_norm = 0.500 +I0915 04:29:39.714215 139606351599360 logging_writer.py:48] [50] global_step=50, grad_norm=0.5, loss=6.72574 +I0915 04:29:39.717356 139627431158976 submission.py:307] 50) loss = 6.726, grad_norm = 0.500 +I0915 04:29:41.526476 139606343206656 logging_writer.py:48] [51] global_step=51, grad_norm=0.5, loss=6.66682 +I0915 04:29:41.529668 139627431158976 submission.py:307] 51) loss = 6.667, grad_norm = 0.500 +I0915 04:29:42.159551 139606351599360 logging_writer.py:48] [52] global_step=52, grad_norm=0.5, loss=6.63948 +I0915 04:29:42.162770 139627431158976 submission.py:307] 52) loss = 6.639, grad_norm = 0.500 +I0915 04:29:42.828228 139606343206656 logging_writer.py:48] [53] global_step=53, grad_norm=0.5, loss=6.59791 +I0915 04:29:42.831326 139627431158976 submission.py:307] 53) loss = 6.598, grad_norm = 0.500 +I0915 04:29:44.891184 139606351599360 logging_writer.py:48] [54] global_step=54, grad_norm=0.5, loss=6.57184 +I0915 04:29:44.894311 139627431158976 submission.py:307] 54) loss = 6.572, grad_norm = 0.500 +I0915 04:29:46.816945 139606343206656 logging_writer.py:48] [55] global_step=55, grad_norm=0.5, loss=6.53727 +I0915 04:29:46.820951 139627431158976 submission.py:307] 55) loss = 6.537, grad_norm = 0.500 +I0915 04:29:47.278398 139606351599360 logging_writer.py:48] [56] global_step=56, grad_norm=0.5, loss=6.49189 +I0915 04:29:47.281467 139627431158976 submission.py:307] 56) loss = 6.492, grad_norm = 0.500 +I0915 04:29:48.450013 139606343206656 logging_writer.py:48] [57] global_step=57, grad_norm=0.5, loss=6.47081 +I0915 04:29:48.453119 139627431158976 submission.py:307] 57) loss = 6.471, grad_norm = 0.500 +I0915 04:29:49.826265 139606351599360 logging_writer.py:48] [58] global_step=58, grad_norm=0.5, loss=6.44724 +I0915 04:29:49.829388 139627431158976 submission.py:307] 58) loss = 6.447, grad_norm = 0.500 +I0915 04:29:51.980522 139606343206656 logging_writer.py:48] [59] global_step=59, grad_norm=0.5, loss=6.43087 +I0915 04:29:51.983728 139627431158976 submission.py:307] 59) loss = 6.431, grad_norm = 0.500 +I0915 04:29:52.461534 139606351599360 logging_writer.py:48] [60] global_step=60, grad_norm=0.5, loss=6.40978 +I0915 04:29:52.464546 139627431158976 submission.py:307] 60) loss = 6.410, grad_norm = 0.500 +I0915 04:29:53.449790 139606343206656 logging_writer.py:48] [61] global_step=61, grad_norm=0.5, loss=6.3813 +I0915 04:29:53.452886 139627431158976 submission.py:307] 61) loss = 6.381, grad_norm = 0.500 +I0915 04:29:55.115748 139606351599360 logging_writer.py:48] [62] global_step=62, grad_norm=0.5, loss=6.36212 +I0915 04:29:55.118818 139627431158976 submission.py:307] 62) loss = 6.362, grad_norm = 0.500 +I0915 04:29:57.303156 139606343206656 logging_writer.py:48] [63] global_step=63, grad_norm=0.5, loss=6.33416 +I0915 04:29:57.306292 139627431158976 submission.py:307] 63) loss = 6.334, grad_norm = 0.500 +I0915 04:29:57.752449 139606351599360 logging_writer.py:48] [64] global_step=64, grad_norm=0.5, loss=6.30517 +I0915 04:29:57.755623 139627431158976 submission.py:307] 64) loss = 6.305, grad_norm = 0.500 +I0915 04:29:58.732674 139606343206656 logging_writer.py:48] [65] global_step=65, grad_norm=0.5, loss=6.31878 +I0915 04:29:58.736171 139627431158976 submission.py:307] 65) loss = 6.319, grad_norm = 0.500 +I0915 04:30:00.374328 139606351599360 logging_writer.py:48] [66] global_step=66, grad_norm=0.5, loss=6.28054 +I0915 04:30:00.377349 139627431158976 submission.py:307] 66) loss = 6.281, grad_norm = 0.500 +I0915 04:30:02.646907 139606343206656 logging_writer.py:48] [67] global_step=67, grad_norm=0.5, loss=6.25406 +I0915 04:30:02.649961 139627431158976 submission.py:307] 67) loss = 6.254, grad_norm = 0.500 +I0915 04:30:03.096824 139606351599360 logging_writer.py:48] [68] global_step=68, grad_norm=0.5, loss=6.2557 +I0915 04:30:03.099955 139627431158976 submission.py:307] 68) loss = 6.256, grad_norm = 0.500 +I0915 04:30:03.932423 139606343206656 logging_writer.py:48] [69] global_step=69, grad_norm=0.5, loss=6.25703 +I0915 04:30:03.935497 139627431158976 submission.py:307] 69) loss = 6.257, grad_norm = 0.500 +I0915 04:30:05.438063 139606351599360 logging_writer.py:48] [70] global_step=70, grad_norm=0.5, loss=6.22837 +I0915 04:30:05.441271 139627431158976 submission.py:307] 70) loss = 6.228, grad_norm = 0.500 +I0915 04:30:07.738345 139606343206656 logging_writer.py:48] [71] global_step=71, grad_norm=0.5, loss=6.2136 +I0915 04:30:07.741801 139627431158976 submission.py:307] 71) loss = 6.214, grad_norm = 0.500 +I0915 04:30:08.416783 139606351599360 logging_writer.py:48] [72] global_step=72, grad_norm=0.5, loss=6.19799 +I0915 04:30:08.419999 139627431158976 submission.py:307] 72) loss = 6.198, grad_norm = 0.500 +I0915 04:30:09.026124 139606343206656 logging_writer.py:48] [73] global_step=73, grad_norm=0.5, loss=6.19057 +I0915 04:30:09.029191 139627431158976 submission.py:307] 73) loss = 6.191, grad_norm = 0.500 +I0915 04:30:10.505617 139606351599360 logging_writer.py:48] [74] global_step=74, grad_norm=0.499999, loss=6.1795 +I0915 04:30:10.508713 139627431158976 submission.py:307] 74) loss = 6.180, grad_norm = 0.500 +I0915 04:30:13.271999 139606343206656 logging_writer.py:48] [75] global_step=75, grad_norm=0.5, loss=6.16969 +I0915 04:30:13.275094 139627431158976 submission.py:307] 75) loss = 6.170, grad_norm = 0.500 +I0915 04:30:14.175264 139606351599360 logging_writer.py:48] [76] global_step=76, grad_norm=0.5, loss=6.15972 +I0915 04:30:14.178354 139627431158976 submission.py:307] 76) loss = 6.160, grad_norm = 0.500 +I0915 04:30:14.624055 139606343206656 logging_writer.py:48] [77] global_step=77, grad_norm=0.5, loss=6.17522 +I0915 04:30:14.627174 139627431158976 submission.py:307] 77) loss = 6.175, grad_norm = 0.500 +I0915 04:30:15.972706 139606351599360 logging_writer.py:48] [78] global_step=78, grad_norm=0.5, loss=6.17708 +I0915 04:30:15.975689 139627431158976 submission.py:307] 78) loss = 6.177, grad_norm = 0.500 +I0915 04:30:18.869293 139606343206656 logging_writer.py:48] [79] global_step=79, grad_norm=0.5, loss=6.13515 +I0915 04:30:18.872407 139627431158976 submission.py:307] 79) loss = 6.135, grad_norm = 0.500 +I0915 04:30:19.318580 139606351599360 logging_writer.py:48] [80] global_step=80, grad_norm=0.5, loss=6.14514 +I0915 04:30:19.321939 139627431158976 submission.py:307] 80) loss = 6.145, grad_norm = 0.500 +I0915 04:30:19.770499 139606343206656 logging_writer.py:48] [81] global_step=81, grad_norm=0.5, loss=6.13768 +I0915 04:30:19.773708 139627431158976 submission.py:307] 81) loss = 6.138, grad_norm = 0.500 +I0915 04:30:21.229439 139606351599360 logging_writer.py:48] [82] global_step=82, grad_norm=0.5, loss=6.09436 +I0915 04:30:21.232599 139627431158976 submission.py:307] 82) loss = 6.094, grad_norm = 0.500 +I0915 04:30:24.762416 139606343206656 logging_writer.py:48] [83] global_step=83, grad_norm=0.5, loss=6.16357 +I0915 04:30:24.767477 139627431158976 submission.py:307] 83) loss = 6.164, grad_norm = 0.500 +I0915 04:30:25.223173 139606351599360 logging_writer.py:48] [84] global_step=84, grad_norm=0.5, loss=6.18582 +I0915 04:30:25.227434 139627431158976 submission.py:307] 84) loss = 6.186, grad_norm = 0.500 +I0915 04:30:25.688554 139606343206656 logging_writer.py:48] [85] global_step=85, grad_norm=0.5, loss=6.12238 +I0915 04:30:25.692590 139627431158976 submission.py:307] 85) loss = 6.122, grad_norm = 0.500 +I0915 04:30:26.745303 139606351599360 logging_writer.py:48] [86] global_step=86, grad_norm=0.5, loss=6.08261 +I0915 04:30:26.748427 139627431158976 submission.py:307] 86) loss = 6.083, grad_norm = 0.500 +I0915 04:30:29.866152 139606343206656 logging_writer.py:48] [87] global_step=87, grad_norm=0.5, loss=6.07266 +I0915 04:30:29.869387 139627431158976 submission.py:307] 87) loss = 6.073, grad_norm = 0.500 +I0915 04:30:30.316138 139606351599360 logging_writer.py:48] [88] global_step=88, grad_norm=0.5, loss=6.08863 +I0915 04:30:30.319610 139627431158976 submission.py:307] 88) loss = 6.089, grad_norm = 0.500 +I0915 04:30:30.767040 139606343206656 logging_writer.py:48] [89] global_step=89, grad_norm=0.5, loss=6.08761 +I0915 04:30:30.770231 139627431158976 submission.py:307] 89) loss = 6.088, grad_norm = 0.500 +I0915 04:30:32.228234 139606351599360 logging_writer.py:48] [90] global_step=90, grad_norm=0.5, loss=6.07428 +I0915 04:30:32.231273 139627431158976 submission.py:307] 90) loss = 6.074, grad_norm = 0.500 +I0915 04:30:35.918355 139606343206656 logging_writer.py:48] [91] global_step=91, grad_norm=0.5, loss=6.05202 +I0915 04:30:35.921413 139627431158976 submission.py:307] 91) loss = 6.052, grad_norm = 0.500 +I0915 04:30:36.367913 139606351599360 logging_writer.py:48] [92] global_step=92, grad_norm=0.5, loss=6.05393 +I0915 04:30:36.370902 139627431158976 submission.py:307] 92) loss = 6.054, grad_norm = 0.500 +I0915 04:30:36.817842 139606343206656 logging_writer.py:48] [93] global_step=93, grad_norm=0.5, loss=6.05692 +I0915 04:30:36.821065 139627431158976 submission.py:307] 93) loss = 6.057, grad_norm = 0.500 +I0915 04:30:37.550313 139606351599360 logging_writer.py:48] [94] global_step=94, grad_norm=0.5, loss=6.03746 +I0915 04:30:37.553413 139627431158976 submission.py:307] 94) loss = 6.037, grad_norm = 0.500 +I0915 04:30:40.880385 139606343206656 logging_writer.py:48] [95] global_step=95, grad_norm=0.499999, loss=6.03286 +I0915 04:30:40.883515 139627431158976 submission.py:307] 95) loss = 6.033, grad_norm = 0.500 +I0915 04:30:41.329722 139606351599360 logging_writer.py:48] [96] global_step=96, grad_norm=0.5, loss=6.03801 +I0915 04:30:41.332686 139627431158976 submission.py:307] 96) loss = 6.038, grad_norm = 0.500 +I0915 04:30:41.778814 139606343206656 logging_writer.py:48] [97] global_step=97, grad_norm=0.5, loss=6.02844 +I0915 04:30:41.781945 139627431158976 submission.py:307] 97) loss = 6.028, grad_norm = 0.500 +I0915 04:30:42.805224 139606351599360 logging_writer.py:48] [98] global_step=98, grad_norm=0.499999, loss=6.00692 +I0915 04:30:42.808300 139627431158976 submission.py:307] 98) loss = 6.007, grad_norm = 0.500 +I0915 04:30:46.361558 139606343206656 logging_writer.py:48] [99] global_step=99, grad_norm=0.5, loss=6.02363 +I0915 04:30:46.364717 139627431158976 submission.py:307] 99) loss = 6.024, grad_norm = 0.500 +I0915 04:30:46.810671 139606351599360 logging_writer.py:48] [100] global_step=100, grad_norm=0.5, loss=6.0192 +I0915 04:30:46.813959 139627431158976 submission.py:307] 100) loss = 6.019, grad_norm = 0.500 +I0915 04:39:49.537436 139606343206656 logging_writer.py:48] [500] global_step=500, grad_norm=0.5, loss=5.79371 +I0915 04:39:49.541714 139627431158976 submission.py:307] 500) loss = 5.794, grad_norm = 0.500 +I0915 04:51:12.913915 139606351599360 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.5, loss=3.5993 +I0915 04:51:12.917803 139627431158976 submission.py:307] 1000) loss = 3.599, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 04:57:47.577629 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +I0915 04:58:04.774838 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 04:58:25.081768 139627431158976 spec.py:363] Evaluating on the test split. +I0915 04:58:35.640321 139627431158976 submission_runner.py:516] Time since start: 1887.57s, Step: 1472, {'train/ctc_loss': 4.09133064106987, 'train/wer': 0.7841118316262344, 'validation/ctc_loss': 4.29811154749347, 'validation/wer': 0.7659440930816396, 'validation/num_examples': 5348, 'test/ctc_loss': 4.081027808245693, 'test/wer': 0.7480957894095424, 'test/num_examples': 2472, 'score': 1770.718579530716, 'total_duration': 1887.5654406547546, 'accumulated_submission_time': 1770.718579530716, 'accumulated_eval_time': 113.29140305519104, 'accumulated_logging_time': 0.05643343925476074} +I0915 04:58:35.677230 139606351599360 logging_writer.py:48] [1472] accumulated_eval_time=113.291, accumulated_logging_time=0.0564334, accumulated_submission_time=1770.72, global_step=1472, preemption_count=0, score=1770.72, test/ctc_loss=4.08103, test/num_examples=2472, test/wer=0.748096, total_duration=1887.57, train/ctc_loss=4.09133, train/wer=0.784112, validation/ctc_loss=4.29811, validation/num_examples=5348, validation/wer=0.765944 +I0915 04:59:02.805449 139606343206656 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.499999, loss=2.73628 +I0915 04:59:02.809128 139627431158976 submission.py:307] 1500) loss = 2.736, grad_norm = 0.500 +I0915 05:09:58.776125 139606351599360 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.499999, loss=2.42189 +I0915 05:09:58.780331 139627431158976 submission.py:307] 2000) loss = 2.422, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 05:15:31.507914 139606351599360 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.499999, loss=2.18211 +I0915 05:15:31.515157 139627431158976 submission.py:307] 2500) loss = 2.182, grad_norm = 0.500 +I0915 05:24:26.608417 139606343206656 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.499999, loss=2.05666 +I0915 05:24:26.612756 139627431158976 submission.py:307] 3000) loss = 2.057, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 05:27:44.139306 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +I0915 05:27:59.920716 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 05:28:19.753970 139627431158976 spec.py:363] Evaluating on the test split. +I0915 05:28:30.656379 139627431158976 submission_runner.py:516] Time since start: 3682.58s, Step: 3276, {'train/ctc_loss': 1.0870527409677266, 'train/wer': 0.347314086400622, 'validation/ctc_loss': 1.3146728859001406, 'validation/wer': 0.3709457828416936, 'validation/num_examples': 5348, 'test/ctc_loss': 1.0242510605522936, 'test/wer': 0.317084069628095, 'test/num_examples': 2472, 'score': 3516.3809208869934, 'total_duration': 3682.5812780857086, 'accumulated_submission_time': 3516.3809208869934, 'accumulated_eval_time': 159.8078715801239, 'accumulated_logging_time': 0.10826325416564941} +I0915 05:28:30.743247 139606351599360 logging_writer.py:48] [3276] accumulated_eval_time=159.808, accumulated_logging_time=0.108263, accumulated_submission_time=3516.38, global_step=3276, preemption_count=0, score=3516.38, test/ctc_loss=1.02425, test/num_examples=2472, test/wer=0.317084, total_duration=3682.58, train/ctc_loss=1.08705, train/wer=0.347314, validation/ctc_loss=1.31467, validation/num_examples=5348, validation/wer=0.370946 +I0915 05:30:32.833309 139606343206656 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.499999, loss=1.91455 +I0915 05:30:32.837138 139627431158976 submission.py:307] 3500) loss = 1.915, grad_norm = 0.500 +I0915 05:38:06.998702 139606351599360 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.499999, loss=1.80962 +I0915 05:38:07.002826 139627431158976 submission.py:307] 4000) loss = 1.810, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 05:43:20.000569 139606351599360 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.499999, loss=1.75912 +I0915 05:43:20.008305 139627431158976 submission.py:307] 4500) loss = 1.759, grad_norm = 0.500 +I0915 05:50:08.501402 139606343206656 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.499999, loss=1.72844 +I0915 05:50:08.505643 139627431158976 submission.py:307] 5000) loss = 1.728, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 05:55:38.742180 139606351599360 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.499999, loss=1.68494 +I0915 05:55:38.749476 139627431158976 submission.py:307] 5500) loss = 1.685, grad_norm = 0.500 +I0915 05:57:39.626038 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 05:57:57.217813 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 05:58:17.093000 139627431158976 spec.py:363] Evaluating on the test split. +I0915 05:58:27.482175 139627431158976 submission_runner.py:516] Time since start: 5479.41s, Step: 5708, {'train/ctc_loss': 0.4708599367690133, 'train/wer': 0.16140858606856112, 'validation/ctc_loss': 0.6882235431422041, 'validation/wer': 0.20584174190122145, 'validation/num_examples': 5348, 'test/ctc_loss': 0.44538866418801215, 'test/wer': 0.1497166534641399, 'test/num_examples': 2472, 'score': 5262.169579744339, 'total_duration': 5479.407527923584, 'accumulated_submission_time': 5262.169579744339, 'accumulated_eval_time': 207.66377234458923, 'accumulated_logging_time': 0.20549464225769043} +I0915 05:58:27.563998 139606351599360 logging_writer.py:48] [5708] accumulated_eval_time=207.664, accumulated_logging_time=0.205495, accumulated_submission_time=5262.17, global_step=5708, preemption_count=0, score=5262.17, test/ctc_loss=0.445389, test/num_examples=2472, test/wer=0.149717, total_duration=5479.41, train/ctc_loss=0.47086, train/wer=0.161409, validation/ctc_loss=0.688224, validation/num_examples=5348, validation/wer=0.205842 +I0915 06:03:00.272164 139606343206656 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.499999, loss=1.63686 +I0915 06:03:00.276172 139627431158976 submission.py:307] 6000) loss = 1.637, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 06:08:40.642841 139606351599360 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.499999, loss=1.56542 +I0915 06:08:40.650527 139627431158976 submission.py:307] 6500) loss = 1.565, grad_norm = 0.500 +I0915 06:14:44.548907 139606343206656 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.499999, loss=1.57662 +I0915 06:14:44.553189 139627431158976 submission.py:307] 7000) loss = 1.577, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 06:20:45.394293 139606351599360 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.499999, loss=1.59037 +I0915 06:20:45.401782 139627431158976 submission.py:307] 7500) loss = 1.590, grad_norm = 0.500 +I0915 06:26:29.827074 139606343206656 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.490957, loss=1.50407 +I0915 06:26:29.831231 139627431158976 submission.py:307] 8000) loss = 1.504, grad_norm = 0.491 +I0915 06:27:38.656060 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 06:27:56.325386 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 06:28:16.620975 139627431158976 spec.py:363] Evaluating on the test split. +I0915 06:28:26.927327 139627431158976 submission_runner.py:516] Time since start: 7278.85s, Step: 8062, {'train/ctc_loss': 0.37399581807097293, 'train/wer': 0.13042163633124018, 'validation/ctc_loss': 0.5951974787145871, 'validation/wer': 0.17736687104716845, 'validation/num_examples': 5348, 'test/ctc_loss': 0.3699733400138719, 'test/wer': 0.12367720837649544, 'test/num_examples': 2472, 'score': 7009.676107883453, 'total_duration': 7278.852687597275, 'accumulated_submission_time': 7009.676107883453, 'accumulated_eval_time': 255.93481731414795, 'accumulated_logging_time': 0.29731035232543945} +I0915 06:28:27.189218 139606351599360 logging_writer.py:48] [8062] accumulated_eval_time=255.935, accumulated_logging_time=0.29731, accumulated_submission_time=7009.68, global_step=8062, preemption_count=0, score=7009.68, test/ctc_loss=0.369973, test/num_examples=2472, test/wer=0.123677, total_duration=7278.85, train/ctc_loss=0.373996, train/wer=0.130422, validation/ctc_loss=0.595197, validation/num_examples=5348, validation/wer=0.177367 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 06:33:39.029715 139606351599360 logging_writer.py:48] [8500] global_step=8500, grad_norm=0.499999, loss=1.49057 +I0915 06:33:39.037384 139627431158976 submission.py:307] 8500) loss = 1.491, grad_norm = 0.500 +I0915 06:39:20.607402 139606343206656 logging_writer.py:48] [9000] global_step=9000, grad_norm=0.499999, loss=1.44115 +I0915 06:39:20.611663 139627431158976 submission.py:307] 9000) loss = 1.441, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 06:45:57.183081 139606351599360 logging_writer.py:48] [9500] global_step=9500, grad_norm=0.499999, loss=1.43393 +I0915 06:45:57.190624 139627431158976 submission.py:307] 9500) loss = 1.434, grad_norm = 0.500 +I0915 06:51:24.300595 139606343206656 logging_writer.py:48] [10000] global_step=10000, grad_norm=0.499999, loss=1.45619 +I0915 06:51:24.304754 139627431158976 submission.py:307] 10000) loss = 1.456, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 06:57:35.002889 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +I0915 06:57:50.519705 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 06:58:10.060272 139627431158976 spec.py:363] Evaluating on the test split. +I0915 06:58:20.376113 139627431158976 submission_runner.py:516] Time since start: 9072.30s, Step: 10391, {'train/ctc_loss': 0.32346017686581297, 'train/wer': 0.11448811330025323, 'validation/ctc_loss': 0.5431776209626783, 'validation/wer': 0.1615314053975764, 'validation/num_examples': 5348, 'test/ctc_loss': 0.3373396317907607, 'test/wer': 0.11138870269940894, 'test/num_examples': 2472, 'score': 8754.697697401047, 'total_duration': 9072.301375627518, 'accumulated_submission_time': 8754.697697401047, 'accumulated_eval_time': 301.30780839920044, 'accumulated_logging_time': 0.5694930553436279} +I0915 06:58:20.488745 139606351599360 logging_writer.py:48] [10391] accumulated_eval_time=301.308, accumulated_logging_time=0.569493, accumulated_submission_time=8754.7, global_step=10391, preemption_count=0, score=8754.7, test/ctc_loss=0.33734, test/num_examples=2472, test/wer=0.111389, total_duration=9072.3, train/ctc_loss=0.32346, train/wer=0.114488, validation/ctc_loss=0.543178, validation/num_examples=5348, validation/wer=0.161531 +I0915 06:59:10.923588 139606343206656 logging_writer.py:48] [10500] global_step=10500, grad_norm=0.499999, loss=1.46066 +I0915 06:59:10.927371 139627431158976 submission.py:307] 10500) loss = 1.461, grad_norm = 0.500 +I0915 07:04:28.732752 139606351599360 logging_writer.py:48] [11000] global_step=11000, grad_norm=0.499999, loss=1.39508 +I0915 07:04:28.736908 139627431158976 submission.py:307] 11000) loss = 1.395, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 07:11:40.333701 139606351599360 logging_writer.py:48] [11500] global_step=11500, grad_norm=0.499999, loss=1.48248 +I0915 07:11:40.341383 139627431158976 submission.py:307] 11500) loss = 1.482, grad_norm = 0.500 +I0915 07:16:28.057710 139606343206656 logging_writer.py:48] [12000] global_step=12000, grad_norm=0.499999, loss=1.36541 +I0915 07:16:28.062061 139627431158976 submission.py:307] 12000) loss = 1.365, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 07:23:53.742413 139606351599360 logging_writer.py:48] [12500] global_step=12500, grad_norm=0.499999, loss=1.322 +I0915 07:23:53.749799 139627431158976 submission.py:307] 12500) loss = 1.322, grad_norm = 0.500 +I0915 07:27:29.951811 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 07:27:46.043636 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 07:28:05.581143 139627431158976 spec.py:363] Evaluating on the test split. +I0915 07:28:15.918789 139627431158976 submission_runner.py:516] Time since start: 10867.84s, Step: 12921, {'train/ctc_loss': 0.2850455873176127, 'train/wer': 0.10295507189253106, 'validation/ctc_loss': 0.5090624882270947, 'validation/wer': 0.15184666634480762, 'validation/num_examples': 5348, 'test/ctc_loss': 0.30628467965675205, 'test/wer': 0.10314220136900047, 'test/num_examples': 2472, 'score': 10500.55568909645, 'total_duration': 10867.844135761261, 'accumulated_submission_time': 10500.55568909645, 'accumulated_eval_time': 347.27461290359497, 'accumulated_logging_time': 0.6946163177490234} +I0915 07:28:16.189203 139606351599360 logging_writer.py:48] [12921] accumulated_eval_time=347.275, accumulated_logging_time=0.694616, accumulated_submission_time=10500.6, global_step=12921, preemption_count=0, score=10500.6, test/ctc_loss=0.306285, test/num_examples=2472, test/wer=0.103142, total_duration=10867.8, train/ctc_loss=0.285046, train/wer=0.102955, validation/ctc_loss=0.509062, validation/num_examples=5348, validation/wer=0.151847 +I0915 07:29:16.964564 139606343206656 logging_writer.py:48] [13000] global_step=13000, grad_norm=0.499999, loss=1.40601 +I0915 07:29:16.969867 139627431158976 submission.py:307] 13000) loss = 1.406, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 07:36:52.881705 139606351599360 logging_writer.py:48] [13500] global_step=13500, grad_norm=0.491844, loss=1.37319 +I0915 07:36:52.891092 139627431158976 submission.py:307] 13500) loss = 1.373, grad_norm = 0.492 +I0915 07:41:16.258948 139606343206656 logging_writer.py:48] [14000] global_step=14000, grad_norm=0.460257, loss=1.41391 +I0915 07:41:16.263369 139627431158976 submission.py:307] 14000) loss = 1.414, grad_norm = 0.460 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 07:49:02.301406 139606351599360 logging_writer.py:48] [14500] global_step=14500, grad_norm=0.485044, loss=1.33105 +I0915 07:49:02.335511 139627431158976 submission.py:307] 14500) loss = 1.331, grad_norm = 0.485 +I0915 07:53:18.360555 139606343206656 logging_writer.py:48] [15000] global_step=15000, grad_norm=0.499999, loss=1.3438 +I0915 07:53:18.364668 139627431158976 submission.py:307] 15000) loss = 1.344, grad_norm = 0.500 +I0915 07:57:25.307266 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 07:57:41.728013 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 07:58:01.431449 139627431158976 spec.py:363] Evaluating on the test split. +I0915 07:58:11.840655 139627431158976 submission_runner.py:516] Time since start: 12663.77s, Step: 15267, {'train/ctc_loss': 0.2590899828182181, 'train/wer': 0.09419190419368598, 'validation/ctc_loss': 0.48662941326979103, 'validation/wer': 0.14627528605223772, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2837675515355829, 'test/wer': 0.09597221375906405, 'test/num_examples': 2472, 'score': 12246.120055437088, 'total_duration': 12663.76597571373, 'accumulated_submission_time': 12246.120055437088, 'accumulated_eval_time': 393.80775809288025, 'accumulated_logging_time': 0.9752578735351562} +I0915 07:58:11.930377 139606351599360 logging_writer.py:48] [15267] accumulated_eval_time=393.808, accumulated_logging_time=0.975258, accumulated_submission_time=12246.1, global_step=15267, preemption_count=0, score=12246.1, test/ctc_loss=0.283768, test/num_examples=2472, test/wer=0.0959722, total_duration=12663.8, train/ctc_loss=0.25909, train/wer=0.0941919, validation/ctc_loss=0.486629, validation/num_examples=5348, validation/wer=0.146275 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 08:01:57.755469 139606351599360 logging_writer.py:48] [15500] global_step=15500, grad_norm=0.479979, loss=1.32105 +I0915 08:01:57.762819 139627431158976 submission.py:307] 15500) loss = 1.321, grad_norm = 0.480 +I0915 08:06:12.646488 139606343206656 logging_writer.py:48] [16000] global_step=16000, grad_norm=0.499999, loss=1.42631 +I0915 08:06:12.650698 139627431158976 submission.py:307] 16000) loss = 1.426, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 08:14:13.497344 139606351599360 logging_writer.py:48] [16500] global_step=16500, grad_norm=0.499999, loss=1.30032 +I0915 08:14:13.504769 139627431158976 submission.py:307] 16500) loss = 1.300, grad_norm = 0.500 +I0915 08:18:13.319099 139606343206656 logging_writer.py:48] [17000] global_step=17000, grad_norm=0.499999, loss=1.27875 +I0915 08:18:13.323207 139627431158976 submission.py:307] 17000) loss = 1.279, grad_norm = 0.500 +I0915 08:26:19.117996 139606351599360 logging_writer.py:48] [17500] global_step=17500, grad_norm=0.499999, loss=1.37211 +I0915 08:26:19.123018 139627431158976 submission.py:307] 17500) loss = 1.372, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 08:27:20.295275 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +I0915 08:27:34.564580 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 08:27:54.124611 139627431158976 spec.py:363] Evaluating on the test split. +I0915 08:28:04.473453 139627431158976 submission_runner.py:516] Time since start: 14456.40s, Step: 17623, {'train/ctc_loss': 0.24585686104403, 'train/wer': 0.09007758886003228, 'validation/ctc_loss': 0.4750910830445047, 'validation/wer': 0.14154395790083524, 'validation/num_examples': 5348, 'test/ctc_loss': 0.28043124758048904, 'test/wer': 0.09215363678833303, 'test/num_examples': 2472, 'score': 13991.609188079834, 'total_duration': 14456.398834943771, 'accumulated_submission_time': 13991.609188079834, 'accumulated_eval_time': 437.9858956336975, 'accumulated_logging_time': 1.0895891189575195} +I0915 08:28:04.755524 139606351599360 logging_writer.py:48] [17623] accumulated_eval_time=437.986, accumulated_logging_time=1.08959, accumulated_submission_time=13991.6, global_step=17623, preemption_count=0, score=13991.6, test/ctc_loss=0.280431, test/num_examples=2472, test/wer=0.0921536, total_duration=14456.4, train/ctc_loss=0.245857, train/wer=0.0900776, validation/ctc_loss=0.475091, validation/num_examples=5348, validation/wer=0.141544 +I0915 08:31:18.136730 139606343206656 logging_writer.py:48] [18000] global_step=18000, grad_norm=0.483368, loss=1.33138 +I0915 08:31:18.141041 139627431158976 submission.py:307] 18000) loss = 1.331, grad_norm = 0.483 +I0915 08:39:11.241951 139606351599360 logging_writer.py:48] [18500] global_step=18500, grad_norm=0.499999, loss=1.34898 +I0915 08:39:11.246151 139627431158976 submission.py:307] 18500) loss = 1.349, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 08:43:26.832187 139606351599360 logging_writer.py:48] [19000] global_step=19000, grad_norm=0.472513, loss=1.30024 +I0915 08:43:26.839807 139627431158976 submission.py:307] 19000) loss = 1.300, grad_norm = 0.473 +I0915 08:51:00.704819 139606343206656 logging_writer.py:48] [19500] global_step=19500, grad_norm=0.481307, loss=1.33423 +I0915 08:51:00.708961 139627431158976 submission.py:307] 19500) loss = 1.334, grad_norm = 0.481 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 08:55:37.768941 139606351599360 logging_writer.py:48] [20000] global_step=20000, grad_norm=0.499999, loss=1.32513 +I0915 08:55:37.776186 139627431158976 submission.py:307] 20000) loss = 1.325, grad_norm = 0.500 +I0915 08:57:14.251336 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 08:57:31.129384 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 08:57:51.016806 139627431158976 spec.py:363] Evaluating on the test split. +I0915 08:58:01.387660 139627431158976 submission_runner.py:516] Time since start: 16253.31s, Step: 20150, {'train/ctc_loss': 0.23004869158025043, 'train/wer': 0.08424627578871209, 'validation/ctc_loss': 0.45861323415837857, 'validation/wer': 0.13579877371698934, 'validation/num_examples': 5348, 'test/ctc_loss': 0.26554031711723336, 'test/wer': 0.08721792293786688, 'test/num_examples': 2472, 'score': 15737.517180681229, 'total_duration': 16253.312993526459, 'accumulated_submission_time': 15737.517180681229, 'accumulated_eval_time': 485.1220736503601, 'accumulated_logging_time': 1.3821239471435547} +I0915 08:58:01.634590 139606351599360 logging_writer.py:48] [20150] accumulated_eval_time=485.122, accumulated_logging_time=1.38212, accumulated_submission_time=15737.5, global_step=20150, preemption_count=0, score=15737.5, test/ctc_loss=0.26554, test/num_examples=2472, test/wer=0.0872179, total_duration=16253.3, train/ctc_loss=0.230049, train/wer=0.0842463, validation/ctc_loss=0.458613, validation/num_examples=5348, validation/wer=0.135799 +I0915 09:03:44.240808 139606343206656 logging_writer.py:48] [20500] global_step=20500, grad_norm=0.469871, loss=1.33779 +I0915 09:03:44.244812 139627431158976 submission.py:307] 20500) loss = 1.338, grad_norm = 0.470 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 09:08:35.290344 139606351599360 logging_writer.py:48] [21000] global_step=21000, grad_norm=0.481668, loss=1.25811 +I0915 09:08:35.307264 139627431158976 submission.py:307] 21000) loss = 1.258, grad_norm = 0.482 +I0915 09:15:29.558916 139606343206656 logging_writer.py:48] [21500] global_step=21500, grad_norm=0.499999, loss=1.31059 +I0915 09:15:29.563133 139627431158976 submission.py:307] 21500) loss = 1.311, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 09:20:42.678136 139606351599360 logging_writer.py:48] [22000] global_step=22000, grad_norm=0.499999, loss=1.28307 +I0915 09:20:42.685431 139627431158976 submission.py:307] 22000) loss = 1.283, grad_norm = 0.500 +I0915 09:27:09.866185 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 09:27:27.252236 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 09:27:47.014701 139627431158976 spec.py:363] Evaluating on the test split. +I0915 09:27:57.075734 139627431158976 submission_runner.py:516] Time since start: 18049.00s, Step: 22490, {'train/ctc_loss': 0.21083678586105079, 'train/wer': 0.07836636844179756, 'validation/ctc_loss': 0.43804812685227046, 'validation/wer': 0.12949355477236518, 'validation/num_examples': 5348, 'test/ctc_loss': 0.25008070077101746, 'test/wer': 0.08262750594113705, 'test/num_examples': 2472, 'score': 17482.665462493896, 'total_duration': 18049.00102376938, 'accumulated_submission_time': 17482.665462493896, 'accumulated_eval_time': 532.3313477039337, 'accumulated_logging_time': 1.639185905456543} +I0915 09:27:57.157608 139606351599360 logging_writer.py:48] [22490] accumulated_eval_time=532.331, accumulated_logging_time=1.63919, accumulated_submission_time=17482.7, global_step=22490, preemption_count=0, score=17482.7, test/ctc_loss=0.250081, test/num_examples=2472, test/wer=0.0826275, total_duration=18049, train/ctc_loss=0.210837, train/wer=0.0783664, validation/ctc_loss=0.438048, validation/num_examples=5348, validation/wer=0.129494 +I0915 09:28:05.082928 139606343206656 logging_writer.py:48] [22500] global_step=22500, grad_norm=0.499999, loss=1.31083 +I0915 09:28:05.086398 139627431158976 submission.py:307] 22500) loss = 1.311, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 09:33:39.000474 139606351599360 logging_writer.py:48] [23000] global_step=23000, grad_norm=0.499999, loss=1.25865 +I0915 09:33:39.007731 139627431158976 submission.py:307] 23000) loss = 1.259, grad_norm = 0.500 +I0915 09:40:07.057063 139606343206656 logging_writer.py:48] [23500] global_step=23500, grad_norm=0.479494, loss=1.25818 +I0915 09:40:07.061207 139627431158976 submission.py:307] 23500) loss = 1.258, grad_norm = 0.479 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 09:45:50.676164 139606351599360 logging_writer.py:48] [24000] global_step=24000, grad_norm=0.499999, loss=1.32617 +I0915 09:45:50.685136 139627431158976 submission.py:307] 24000) loss = 1.326, grad_norm = 0.500 +I0915 09:51:51.740560 139606343206656 logging_writer.py:48] [24500] global_step=24500, grad_norm=0.470353, loss=1.27054 +I0915 09:51:51.744861 139627431158976 submission.py:307] 24500) loss = 1.271, grad_norm = 0.470 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 09:57:05.453212 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +I0915 09:57:19.608463 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 09:57:39.467782 139627431158976 spec.py:363] Evaluating on the test split. +I0915 09:57:49.788754 139627431158976 submission_runner.py:516] Time since start: 19841.71s, Step: 24869, {'train/ctc_loss': 0.1999121621564489, 'train/wer': 0.07424665374418893, 'validation/ctc_loss': 0.42751642908943643, 'validation/wer': 0.12612369043595809, 'validation/num_examples': 5348, 'test/ctc_loss': 0.24434197367572102, 'test/wer': 0.08149005748177035, 'test/num_examples': 2472, 'score': 19228.162989377975, 'total_duration': 19841.71408224106, 'accumulated_submission_time': 19228.162989377975, 'accumulated_eval_time': 576.6667482852936, 'accumulated_logging_time': 1.7311475276947021} +I0915 09:57:50.050054 139606351599360 logging_writer.py:48] [24869] accumulated_eval_time=576.667, accumulated_logging_time=1.73115, accumulated_submission_time=19228.2, global_step=24869, preemption_count=0, score=19228.2, test/ctc_loss=0.244342, test/num_examples=2472, test/wer=0.0814901, total_duration=19841.7, train/ctc_loss=0.199912, train/wer=0.0742467, validation/ctc_loss=0.427516, validation/num_examples=5348, validation/wer=0.126124 +I0915 09:58:49.717637 139606343206656 logging_writer.py:48] [25000] global_step=25000, grad_norm=0.499999, loss=1.27286 +I0915 09:58:49.720988 139627431158976 submission.py:307] 25000) loss = 1.273, grad_norm = 0.500 +I0915 10:04:49.105767 139606351599360 logging_writer.py:48] [25500] global_step=25500, grad_norm=0.499999, loss=1.20204 +I0915 10:04:49.109922 139627431158976 submission.py:307] 25500) loss = 1.202, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 10:11:15.723195 139606351599360 logging_writer.py:48] [26000] global_step=26000, grad_norm=0.435657, loss=1.20735 +I0915 10:11:15.730640 139627431158976 submission.py:307] 26000) loss = 1.207, grad_norm = 0.436 +I0915 10:16:40.010351 139606343206656 logging_writer.py:48] [26500] global_step=26500, grad_norm=0.499999, loss=1.29912 +I0915 10:16:40.014477 139627431158976 submission.py:307] 26500) loss = 1.299, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 10:23:29.325191 139606351599360 logging_writer.py:48] [27000] global_step=27000, grad_norm=0.499999, loss=1.25499 +I0915 10:23:29.332903 139627431158976 submission.py:307] 27000) loss = 1.255, grad_norm = 0.500 +I0915 10:26:58.716797 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 10:27:14.486836 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 10:27:34.425461 139627431158976 spec.py:363] Evaluating on the test split. +I0915 10:27:44.790097 139627431158976 submission_runner.py:516] Time since start: 21636.72s, Step: 27381, {'train/ctc_loss': 0.19035911625859936, 'train/wer': 0.07099083727936849, 'validation/ctc_loss': 0.42490220639943743, 'validation/wer': 0.12364215709940617, 'validation/num_examples': 5348, 'test/ctc_loss': 0.23842490846183625, 'test/wer': 0.07828082789998578, 'test/num_examples': 2472, 'score': 20973.073004722595, 'total_duration': 21636.715457201004, 'accumulated_submission_time': 20973.073004722595, 'accumulated_eval_time': 622.7398328781128, 'accumulated_logging_time': 2.002565860748291} +I0915 10:27:45.031729 139606351599360 logging_writer.py:48] [27381] accumulated_eval_time=622.74, accumulated_logging_time=2.00257, accumulated_submission_time=20973.1, global_step=27381, preemption_count=0, score=20973.1, test/ctc_loss=0.238425, test/num_examples=2472, test/wer=0.0782808, total_duration=21636.7, train/ctc_loss=0.190359, train/wer=0.0709908, validation/ctc_loss=0.424902, validation/num_examples=5348, validation/wer=0.123642 +I0915 10:29:26.291517 139606343206656 logging_writer.py:48] [27500] global_step=27500, grad_norm=0.499999, loss=1.19015 +I0915 10:29:26.295554 139627431158976 submission.py:307] 27500) loss = 1.190, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 10:36:23.100380 139606351599360 logging_writer.py:48] [28000] global_step=28000, grad_norm=0.470654, loss=1.26911 +I0915 10:36:23.108413 139627431158976 submission.py:307] 28000) loss = 1.269, grad_norm = 0.471 +I0915 10:41:20.602324 139606343206656 logging_writer.py:48] [28500] global_step=28500, grad_norm=0.499999, loss=1.1536 +I0915 10:41:20.606488 139627431158976 submission.py:307] 28500) loss = 1.154, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 10:48:34.943158 139606351599360 logging_writer.py:48] [29000] global_step=29000, grad_norm=0.499999, loss=1.22059 +I0915 10:48:34.950593 139627431158976 submission.py:307] 29000) loss = 1.221, grad_norm = 0.500 +I0915 10:53:16.920208 139606343206656 logging_writer.py:48] [29500] global_step=29500, grad_norm=0.499999, loss=1.20985 +I0915 10:53:16.924347 139627431158976 submission.py:307] 29500) loss = 1.210, grad_norm = 0.500 +I0915 10:56:53.264349 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 10:57:08.156998 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 10:57:27.798306 139627431158976 spec.py:363] Evaluating on the test split. +I0915 10:57:38.099780 139627431158976 submission_runner.py:516] Time since start: 23430.03s, Step: 29716, {'train/ctc_loss': 0.17910000746206817, 'train/wer': 0.0671032952318217, 'validation/ctc_loss': 0.40949726162221217, 'validation/wer': 0.11796456331772318, 'validation/num_examples': 5348, 'test/ctc_loss': 0.23190142811633008, 'test/wer': 0.07517315621635895, 'test/num_examples': 2472, 'score': 22718.525512218475, 'total_duration': 23430.025129795074, 'accumulated_submission_time': 22718.525512218475, 'accumulated_eval_time': 667.5750579833984, 'accumulated_logging_time': 2.254542827606201} +I0915 10:57:38.187196 139606351599360 logging_writer.py:48] [29716] accumulated_eval_time=667.575, accumulated_logging_time=2.25454, accumulated_submission_time=22718.5, global_step=29716, preemption_count=0, score=22718.5, test/ctc_loss=0.231901, test/num_examples=2472, test/wer=0.0751732, total_duration=23430, train/ctc_loss=0.1791, train/wer=0.0671033, validation/ctc_loss=0.409497, validation/num_examples=5348, validation/wer=0.117965 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 11:01:26.793717 139606351599360 logging_writer.py:48] [30000] global_step=30000, grad_norm=0.4957, loss=1.20415 +I0915 11:01:26.807187 139627431158976 submission.py:307] 30000) loss = 1.204, grad_norm = 0.496 +I0915 11:06:10.398352 139606343206656 logging_writer.py:48] [30500] global_step=30500, grad_norm=0.499999, loss=1.24732 +I0915 11:06:10.403285 139627431158976 submission.py:307] 30500) loss = 1.247, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 11:13:44.625910 139606351599360 logging_writer.py:48] [31000] global_step=31000, grad_norm=0.369589, loss=1.16144 +I0915 11:13:44.633943 139627431158976 submission.py:307] 31000) loss = 1.161, grad_norm = 0.370 +I0915 11:18:05.840725 139606343206656 logging_writer.py:48] [31500] global_step=31500, grad_norm=0.406493, loss=1.25011 +I0915 11:18:05.844953 139627431158976 submission.py:307] 31500) loss = 1.250, grad_norm = 0.406 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 11:25:59.859015 139606351599360 logging_writer.py:48] [32000] global_step=32000, grad_norm=0.499999, loss=1.2171 +I0915 11:25:59.873276 139627431158976 submission.py:307] 32000) loss = 1.217, grad_norm = 0.500 +I0915 11:26:46.407405 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 11:27:02.686383 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 11:27:22.503061 139627431158976 spec.py:363] Evaluating on the test split. +I0915 11:27:33.067711 139627431158976 submission_runner.py:516] Time since start: 25224.99s, Step: 32106, {'train/ctc_loss': 0.17202233885314988, 'train/wer': 0.06474377318351898, 'validation/ctc_loss': 0.40757545745585944, 'validation/wer': 0.11669965722010332, 'validation/num_examples': 5348, 'test/ctc_loss': 0.22778813350861346, 'test/wer': 0.0731216866735726, 'test/num_examples': 2472, 'score': 24463.878875732422, 'total_duration': 25224.993047475815, 'accumulated_submission_time': 24463.878875732422, 'accumulated_eval_time': 714.2352414131165, 'accumulated_logging_time': 2.3517918586730957} +I0915 11:27:33.369712 139606351599360 logging_writer.py:48] [32106] accumulated_eval_time=714.235, accumulated_logging_time=2.35179, accumulated_submission_time=24463.9, global_step=32106, preemption_count=0, score=24463.9, test/ctc_loss=0.227788, test/num_examples=2472, test/wer=0.0731217, total_duration=25225, train/ctc_loss=0.172022, train/wer=0.0647438, validation/ctc_loss=0.407575, validation/num_examples=5348, validation/wer=0.1167 +I0915 11:31:12.649394 139606343206656 logging_writer.py:48] [32500] global_step=32500, grad_norm=0.476968, loss=1.15463 +I0915 11:31:12.653811 139627431158976 submission.py:307] 32500) loss = 1.155, grad_norm = 0.477 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 11:39:12.708374 139606351599360 logging_writer.py:48] [33000] global_step=33000, grad_norm=0.499999, loss=1.15292 +I0915 11:39:12.715931 139627431158976 submission.py:307] 33000) loss = 1.153, grad_norm = 0.500 +I0915 11:43:15.590005 139606343206656 logging_writer.py:48] [33500] global_step=33500, grad_norm=0.45754, loss=1.1466 +I0915 11:43:15.594558 139627431158976 submission.py:307] 33500) loss = 1.147, grad_norm = 0.458 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 11:51:25.588000 139606351599360 logging_writer.py:48] [34000] global_step=34000, grad_norm=0.499999, loss=1.11145 +I0915 11:51:25.595436 139627431158976 submission.py:307] 34000) loss = 1.111, grad_norm = 0.500 +I0915 11:55:22.103139 139606343206656 logging_writer.py:48] [34500] global_step=34500, grad_norm=0.499999, loss=1.19437 +I0915 11:55:22.107893 139627431158976 submission.py:307] 34500) loss = 1.194, grad_norm = 0.500 +I0915 11:56:42.147178 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 11:56:58.569423 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 11:57:18.319675 139627431158976 spec.py:363] Evaluating on the test split. +I0915 11:57:28.698077 139627431158976 submission_runner.py:516] Time since start: 27020.62s, Step: 34613, {'train/ctc_loss': 0.16054282952323937, 'train/wer': 0.061131598697673414, 'validation/ctc_loss': 0.39606849711799275, 'validation/wer': 0.113204267851108, 'validation/num_examples': 5348, 'test/ctc_loss': 0.22421025650848442, 'test/wer': 0.0711311518696809, 'test/num_examples': 2472, 'score': 26208.83354616165, 'total_duration': 27020.623435735703, 'accumulated_submission_time': 26208.83354616165, 'accumulated_eval_time': 760.7859456539154, 'accumulated_logging_time': 2.663914442062378} +I0915 11:57:29.032935 139606351599360 logging_writer.py:48] [34613] accumulated_eval_time=760.786, accumulated_logging_time=2.66391, accumulated_submission_time=26208.8, global_step=34613, preemption_count=0, score=26208.8, test/ctc_loss=0.22421, test/num_examples=2472, test/wer=0.0711312, total_duration=27020.6, train/ctc_loss=0.160543, train/wer=0.0611316, validation/ctc_loss=0.396068, validation/num_examples=5348, validation/wer=0.113204 +I0915 12:04:10.415321 139606343206656 logging_writer.py:48] [35000] global_step=35000, grad_norm=0.499999, loss=1.15977 +I0915 12:04:10.419425 139627431158976 submission.py:307] 35000) loss = 1.160, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 12:08:16.400022 139606351599360 logging_writer.py:48] [35500] global_step=35500, grad_norm=0.352559, loss=1.05314 +I0915 12:08:16.407447 139627431158976 submission.py:307] 35500) loss = 1.053, grad_norm = 0.353 +I0915 12:15:58.709420 139606343206656 logging_writer.py:48] [36000] global_step=36000, grad_norm=0.44861, loss=1.14716 +I0915 12:15:58.713780 139627431158976 submission.py:307] 36000) loss = 1.147, grad_norm = 0.449 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 12:20:22.666501 139606351599360 logging_writer.py:48] [36500] global_step=36500, grad_norm=0.499999, loss=1.1026 +I0915 12:20:22.674157 139627431158976 submission.py:307] 36500) loss = 1.103, grad_norm = 0.500 +I0915 12:26:37.083057 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 12:26:53.723218 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 12:27:13.680088 139627431158976 spec.py:363] Evaluating on the test split. +I0915 12:27:23.973212 139627431158976 submission_runner.py:516] Time since start: 28815.90s, Step: 36937, {'train/ctc_loss': 0.1492908947759604, 'train/wer': 0.05713067000707317, 'validation/ctc_loss': 0.3844368666176914, 'validation/wer': 0.10995027277555158, 'validation/num_examples': 5348, 'test/ctc_loss': 0.21279397057874702, 'test/wer': 0.06836877703979038, 'test/num_examples': 2472, 'score': 27953.990550994873, 'total_duration': 28815.898555278778, 'accumulated_submission_time': 27953.990550994873, 'accumulated_eval_time': 807.6759321689606, 'accumulated_logging_time': 3.0088658332824707} +I0915 12:27:24.034896 139606351599360 logging_writer.py:48] [36937] accumulated_eval_time=807.676, accumulated_logging_time=3.00887, accumulated_submission_time=27954, global_step=36937, preemption_count=0, score=27954, test/ctc_loss=0.212794, test/num_examples=2472, test/wer=0.0683688, total_duration=28815.9, train/ctc_loss=0.149291, train/wer=0.0571307, validation/ctc_loss=0.384437, validation/num_examples=5348, validation/wer=0.10995 +I0915 12:28:31.672855 139606343206656 logging_writer.py:48] [37000] global_step=37000, grad_norm=0.399856, loss=1.07376 +I0915 12:28:31.676354 139627431158976 submission.py:307] 37000) loss = 1.074, grad_norm = 0.400 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 12:33:16.975253 139606351599360 logging_writer.py:48] [37500] global_step=37500, grad_norm=0.499999, loss=1.13245 +I0915 12:33:16.983508 139627431158976 submission.py:307] 37500) loss = 1.132, grad_norm = 0.500 +I0915 12:40:28.748833 139606343206656 logging_writer.py:48] [38000] global_step=38000, grad_norm=0.499999, loss=1.13901 +I0915 12:40:28.752951 139627431158976 submission.py:307] 38000) loss = 1.139, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 12:45:28.533440 139606351599360 logging_writer.py:48] [38500] global_step=38500, grad_norm=0.499999, loss=1.11655 +I0915 12:45:28.540628 139627431158976 submission.py:307] 38500) loss = 1.117, grad_norm = 0.500 +I0915 12:52:21.928185 139606343206656 logging_writer.py:48] [39000] global_step=39000, grad_norm=0.499999, loss=1.1405 +I0915 12:52:21.932294 139627431158976 submission.py:307] 39000) loss = 1.140, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 12:56:32.673526 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +I0915 12:56:50.510698 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 12:57:10.323487 139627431158976 spec.py:363] Evaluating on the test split. +I0915 12:57:20.689525 139627431158976 submission_runner.py:516] Time since start: 30612.61s, Step: 39346, {'train/ctc_loss': 0.14197608618540766, 'train/wer': 0.05470095622735642, 'validation/ctc_loss': 0.3875349802450648, 'validation/wer': 0.10943851687346111, 'validation/num_examples': 5348, 'test/ctc_loss': 0.21229305620523906, 'test/wer': 0.06585014116547844, 'test/num_examples': 2472, 'score': 29699.269289255142, 'total_duration': 30612.614881038666, 'accumulated_submission_time': 29699.269289255142, 'accumulated_eval_time': 855.6917436122894, 'accumulated_logging_time': 3.0807156562805176} +I0915 12:57:20.948040 139606351599360 logging_writer.py:48] [39346] accumulated_eval_time=855.692, accumulated_logging_time=3.08072, accumulated_submission_time=29699.3, global_step=39346, preemption_count=0, score=29699.3, test/ctc_loss=0.212293, test/num_examples=2472, test/wer=0.0658501, total_duration=30612.6, train/ctc_loss=0.141976, train/wer=0.054701, validation/ctc_loss=0.387535, validation/num_examples=5348, validation/wer=0.109439 +I0915 12:58:35.565423 139606343206656 logging_writer.py:48] [39500] global_step=39500, grad_norm=0.499999, loss=1.16441 +I0915 12:58:35.569038 139627431158976 submission.py:307] 39500) loss = 1.164, grad_norm = 0.500 +I0915 13:05:20.898813 139606351599360 logging_writer.py:48] [40000] global_step=40000, grad_norm=0.499999, loss=1.11273 +I0915 13:05:20.902921 139627431158976 submission.py:307] 40000) loss = 1.113, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 13:10:54.640221 139606351599360 logging_writer.py:48] [40500] global_step=40500, grad_norm=0.402405, loss=1.10212 +I0915 13:10:54.647683 139627431158976 submission.py:307] 40500) loss = 1.102, grad_norm = 0.402 +I0915 13:17:08.438306 139606343206656 logging_writer.py:48] [41000] global_step=41000, grad_norm=0.429694, loss=1.08755 +I0915 13:17:08.444160 139627431158976 submission.py:307] 41000) loss = 1.088, grad_norm = 0.430 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 13:23:05.771466 139606351599360 logging_writer.py:48] [41500] global_step=41500, grad_norm=0.486747, loss=1.11678 +I0915 13:23:05.779049 139627431158976 submission.py:307] 41500) loss = 1.117, grad_norm = 0.487 +I0915 13:26:30.462980 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 13:26:48.795378 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 13:27:08.713816 139627431158976 spec.py:363] Evaluating on the test split. +I0915 13:27:19.147528 139627431158976 submission_runner.py:516] Time since start: 32411.07s, Step: 41839, {'train/ctc_loss': 0.13512448600513424, 'train/wer': 0.05243862273024238, 'validation/ctc_loss': 0.37879244524814143, 'validation/wer': 0.10716941051513543, 'validation/num_examples': 5348, 'test/ctc_loss': 0.20776986872136913, 'test/wer': 0.06432677269311235, 'test/num_examples': 2472, 'score': 31445.309044122696, 'total_duration': 32411.072816848755, 'accumulated_submission_time': 31445.309044122696, 'accumulated_eval_time': 904.376006603241, 'accumulated_logging_time': 3.3495373725891113} +I0915 13:27:19.456376 139606351599360 logging_writer.py:48] [41839] accumulated_eval_time=904.376, accumulated_logging_time=3.34954, accumulated_submission_time=31445.3, global_step=41839, preemption_count=0, score=31445.3, test/ctc_loss=0.20777, test/num_examples=2472, test/wer=0.0643268, total_duration=32411.1, train/ctc_loss=0.135124, train/wer=0.0524386, validation/ctc_loss=0.378792, validation/num_examples=5348, validation/wer=0.107169 +I0915 13:29:53.701126 139606343206656 logging_writer.py:48] [42000] global_step=42000, grad_norm=0.499999, loss=1.12544 +I0915 13:29:53.705374 139627431158976 submission.py:307] 42000) loss = 1.125, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 13:36:03.743567 139606351599360 logging_writer.py:48] [42500] global_step=42500, grad_norm=0.499999, loss=1.04256 +I0915 13:36:03.751127 139627431158976 submission.py:307] 42500) loss = 1.043, grad_norm = 0.500 +I0915 13:41:43.240767 139606343206656 logging_writer.py:48] [43000] global_step=43000, grad_norm=0.499999, loss=1.0839 +I0915 13:41:43.247208 139627431158976 submission.py:307] 43000) loss = 1.084, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 13:48:15.921335 139606351599360 logging_writer.py:48] [43500] global_step=43500, grad_norm=0.499999, loss=1.073 +I0915 13:48:15.931107 139627431158976 submission.py:307] 43500) loss = 1.073, grad_norm = 0.500 +I0915 13:53:36.146579 139606343206656 logging_writer.py:48] [44000] global_step=44000, grad_norm=0.499999, loss=1.08773 +I0915 13:53:36.150995 139627431158976 submission.py:307] 44000) loss = 1.088, grad_norm = 0.500 +I0915 13:56:28.098202 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 13:56:43.324807 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 13:57:03.137793 139627431158976 spec.py:363] Evaluating on the test split. +I0915 13:57:13.608507 139627431158976 submission_runner.py:516] Time since start: 34205.53s, Step: 44160, {'train/ctc_loss': 0.12793783262221725, 'train/wer': 0.0490100266188643, 'validation/ctc_loss': 0.36792249719804854, 'validation/wer': 0.10269878820064694, 'validation/num_examples': 5348, 'test/ctc_loss': 0.20071557035937804, 'test/wer': 0.06318932423374565, 'test/num_examples': 2472, 'score': 33191.16186285019, 'total_duration': 34205.5338447094, 'accumulated_submission_time': 33191.16186285019, 'accumulated_eval_time': 949.8861050605774, 'accumulated_logging_time': 3.668992757797241} +I0915 13:57:13.677402 139606351599360 logging_writer.py:48] [44160] accumulated_eval_time=949.886, accumulated_logging_time=3.66899, accumulated_submission_time=33191.2, global_step=44160, preemption_count=0, score=33191.2, test/ctc_loss=0.200716, test/num_examples=2472, test/wer=0.0631893, total_duration=34205.5, train/ctc_loss=0.127938, train/wer=0.04901, validation/ctc_loss=0.367922, validation/num_examples=5348, validation/wer=0.102699 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 14:01:10.737441 139606351599360 logging_writer.py:48] [44500] global_step=44500, grad_norm=0.488758, loss=1.10748 +I0915 14:01:10.744911 139627431158976 submission.py:307] 44500) loss = 1.107, grad_norm = 0.489 +I0915 14:06:27.260293 139606343206656 logging_writer.py:48] [45000] global_step=45000, grad_norm=0.5, loss=1.0561 +I0915 14:06:27.264576 139627431158976 submission.py:307] 45000) loss = 1.056, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 14:13:29.456223 139606351599360 logging_writer.py:48] [45500] global_step=45500, grad_norm=0.499999, loss=1.02165 +I0915 14:13:29.463885 139627431158976 submission.py:307] 45500) loss = 1.022, grad_norm = 0.500 +I0915 14:18:22.974509 139606343206656 logging_writer.py:48] [46000] global_step=46000, grad_norm=0.499999, loss=1.06413 +I0915 14:18:22.978672 139627431158976 submission.py:307] 46000) loss = 1.064, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 14:25:43.824794 139606351599360 logging_writer.py:48] [46500] global_step=46500, grad_norm=0.499999, loss=1.0856 +I0915 14:25:43.833405 139627431158976 submission.py:307] 46500) loss = 1.086, grad_norm = 0.500 +I0915 14:26:22.308488 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 14:26:37.860511 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 14:26:57.584174 139627431158976 spec.py:363] Evaluating on the test split. +I0915 14:27:08.047353 139627431158976 submission_runner.py:516] Time since start: 35999.97s, Step: 46586, {'train/ctc_loss': 0.12093888037691689, 'train/wer': 0.04720663905791898, 'validation/ctc_loss': 0.3711306384368093, 'validation/wer': 0.10291121517887317, 'validation/num_examples': 5348, 'test/ctc_loss': 0.19896812899219304, 'test/wer': 0.06229561472995755, 'test/num_examples': 2472, 'score': 34936.36906123161, 'total_duration': 35999.97264456749, 'accumulated_submission_time': 34936.36906123161, 'accumulated_eval_time': 995.6247532367706, 'accumulated_logging_time': 3.748096227645874} +I0915 14:27:08.310504 139606351599360 logging_writer.py:48] [46586] accumulated_eval_time=995.625, accumulated_logging_time=3.7481, accumulated_submission_time=34936.4, global_step=46586, preemption_count=0, score=34936.4, test/ctc_loss=0.198968, test/num_examples=2472, test/wer=0.0622956, total_duration=36000, train/ctc_loss=0.120939, train/wer=0.0472066, validation/ctc_loss=0.371131, validation/num_examples=5348, validation/wer=0.102911 +I0915 14:31:25.917593 139606343206656 logging_writer.py:48] [47000] global_step=47000, grad_norm=0.499999, loss=1.13461 +I0915 14:31:25.922427 139627431158976 submission.py:307] 47000) loss = 1.135, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 14:38:57.429052 139606351599360 logging_writer.py:48] [47500] global_step=47500, grad_norm=0.499999, loss=1.07371 +I0915 14:38:57.436579 139627431158976 submission.py:307] 47500) loss = 1.074, grad_norm = 0.500 +I0915 14:43:22.840187 139606343206656 logging_writer.py:48] [48000] global_step=48000, grad_norm=0.499999, loss=1.08534 +I0915 14:43:22.844333 139627431158976 submission.py:307] 48000) loss = 1.085, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 14:51:06.690937 139606351599360 logging_writer.py:48] [48500] global_step=48500, grad_norm=0.499999, loss=1.04021 +I0915 14:51:06.698405 139627431158976 submission.py:307] 48500) loss = 1.040, grad_norm = 0.500 +I0915 14:55:22.269107 139606343206656 logging_writer.py:48] [49000] global_step=49000, grad_norm=0.499999, loss=1.0547 +I0915 14:55:22.273252 139627431158976 submission.py:307] 49000) loss = 1.055, grad_norm = 0.500 +I0915 14:56:17.181607 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 14:56:33.214594 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 14:56:53.001624 139627431158976 spec.py:363] Evaluating on the test split. +I0915 14:57:03.517915 139627431158976 submission_runner.py:516] Time since start: 37795.44s, Step: 49071, {'train/ctc_loss': 0.1120992890530403, 'train/wer': 0.043923825773323905, 'validation/ctc_loss': 0.35817573337351816, 'validation/wer': 0.09868198715782359, 'validation/num_examples': 5348, 'test/ctc_loss': 0.1923740165696174, 'test/wer': 0.06000040623159263, 'test/num_examples': 2472, 'score': 36681.68979883194, 'total_duration': 37795.44325685501, 'accumulated_submission_time': 36681.68979883194, 'accumulated_eval_time': 1041.9608659744263, 'accumulated_logging_time': 4.022969007492065} +I0915 14:57:03.784553 139606351599360 logging_writer.py:48] [49071] accumulated_eval_time=1041.96, accumulated_logging_time=4.02297, accumulated_submission_time=36681.7, global_step=49071, preemption_count=0, score=36681.7, test/ctc_loss=0.192374, test/num_examples=2472, test/wer=0.0600004, total_duration=37795.4, train/ctc_loss=0.112099, train/wer=0.0439238, validation/ctc_loss=0.358176, validation/num_examples=5348, validation/wer=0.098682 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 15:03:58.281739 139606351599360 logging_writer.py:48] [49500] global_step=49500, grad_norm=0.499999, loss=1.03517 +I0915 15:03:58.289115 139627431158976 submission.py:307] 49500) loss = 1.035, grad_norm = 0.500 +I0915 15:08:11.594198 139606343206656 logging_writer.py:48] [50000] global_step=50000, grad_norm=0.499999, loss=1.05224 +I0915 15:08:11.598304 139627431158976 submission.py:307] 50000) loss = 1.052, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 15:16:07.558155 139606351599360 logging_writer.py:48] [50500] global_step=50500, grad_norm=0.499999, loss=1.032 +I0915 15:16:07.565891 139627431158976 submission.py:307] 50500) loss = 1.032, grad_norm = 0.500 +I0915 15:20:08.516720 139606343206656 logging_writer.py:48] [51000] global_step=51000, grad_norm=0.499999, loss=1.03971 +I0915 15:20:08.520897 139627431158976 submission.py:307] 51000) loss = 1.040, grad_norm = 0.500 +I0915 15:26:14.451634 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 15:26:29.379068 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 15:26:48.967148 139627431158976 spec.py:363] Evaluating on the test split. +I0915 15:26:59.284751 139627431158976 submission_runner.py:516] Time since start: 39591.21s, Step: 51393, {'train/ctc_loss': 0.10544510074003423, 'train/wer': 0.04147791390174237, 'validation/ctc_loss': 0.3509819976579767, 'validation/wer': 0.0967218654950997, 'validation/num_examples': 5348, 'test/ctc_loss': 0.1892098255330989, 'test/wer': 0.05792862510917474, 'test/num_examples': 2472, 'score': 38429.587532520294, 'total_duration': 39591.210107803345, 'accumulated_submission_time': 38429.587532520294, 'accumulated_eval_time': 1086.793828010559, 'accumulated_logging_time': 4.299762964248657} +I0915 15:26:59.371537 139606351599360 logging_writer.py:48] [51393] accumulated_eval_time=1086.79, accumulated_logging_time=4.29976, accumulated_submission_time=38429.6, global_step=51393, preemption_count=0, score=38429.6, test/ctc_loss=0.18921, test/num_examples=2472, test/wer=0.0579286, total_duration=39591.2, train/ctc_loss=0.105445, train/wer=0.0414779, validation/ctc_loss=0.350982, validation/num_examples=5348, validation/wer=0.0967219 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 15:28:59.935817 139606351599360 logging_writer.py:48] [51500] global_step=51500, grad_norm=0.499999, loss=0.98894 +I0915 15:28:59.945049 139627431158976 submission.py:307] 51500) loss = 0.989, grad_norm = 0.500 +I0915 15:33:03.363773 139606343206656 logging_writer.py:48] [52000] global_step=52000, grad_norm=0.499999, loss=0.941541 +I0915 15:33:03.368146 139627431158976 submission.py:307] 52000) loss = 0.942, grad_norm = 0.500 +I0915 15:40:59.343203 139606351599360 logging_writer.py:48] [52500] global_step=52500, grad_norm=0.499999, loss=1.00007 +I0915 15:40:59.347736 139627431158976 submission.py:307] 52500) loss = 1.000, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 15:45:08.968293 139606351599360 logging_writer.py:48] [53000] global_step=53000, grad_norm=0.499999, loss=1.07498 +I0915 15:45:08.975632 139627431158976 submission.py:307] 53000) loss = 1.075, grad_norm = 0.500 +I0915 15:52:50.611499 139606343206656 logging_writer.py:48] [53500] global_step=53500, grad_norm=0.499999, loss=1.02539 +I0915 15:52:50.615674 139627431158976 submission.py:307] 53500) loss = 1.025, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 15:56:07.953025 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +I0915 15:56:23.288164 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 15:56:43.139164 139627431158976 spec.py:363] Evaluating on the test split. +I0915 15:56:53.455038 139627431158976 submission_runner.py:516] Time since start: 41385.38s, Step: 53851, {'train/ctc_loss': 0.09781304195119765, 'train/wer': 0.03890781665919755, 'validation/ctc_loss': 0.3491209956424553, 'validation/wer': 0.09551489402790518, 'validation/num_examples': 5348, 'test/ctc_loss': 0.18438112438705723, 'test/wer': 0.05626307557938781, 'test/num_examples': 2472, 'score': 40174.67117357254, 'total_duration': 41385.38038396835, 'accumulated_submission_time': 40174.67117357254, 'accumulated_eval_time': 1132.2957038879395, 'accumulated_logging_time': 4.396579027175903} +I0915 15:56:53.768591 139606351599360 logging_writer.py:48] [53851] accumulated_eval_time=1132.3, accumulated_logging_time=4.39658, accumulated_submission_time=40174.7, global_step=53851, preemption_count=0, score=40174.7, test/ctc_loss=0.184381, test/num_examples=2472, test/wer=0.0562631, total_duration=41385.4, train/ctc_loss=0.097813, train/wer=0.0389078, validation/ctc_loss=0.349121, validation/num_examples=5348, validation/wer=0.0955149 +I0915 15:58:15.986100 139606343206656 logging_writer.py:48] [54000] global_step=54000, grad_norm=0.499999, loss=1.06568 +I0915 15:58:15.990000 139627431158976 submission.py:307] 54000) loss = 1.066, grad_norm = 0.500 +I0915 16:05:42.389638 139606351599360 logging_writer.py:48] [54500] global_step=54500, grad_norm=0.499999, loss=1.0323 +I0915 16:05:42.393699 139627431158976 submission.py:307] 54500) loss = 1.032, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 16:10:29.447521 139606351599360 logging_writer.py:48] [55000] global_step=55000, grad_norm=0.499999, loss=1.03323 +I0915 16:10:29.455024 139627431158976 submission.py:307] 55000) loss = 1.033, grad_norm = 0.500 +I0915 16:17:31.520413 139606343206656 logging_writer.py:48] [55500] global_step=55500, grad_norm=0.499999, loss=0.974049 +I0915 16:17:31.524758 139627431158976 submission.py:307] 55500) loss = 0.974, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 16:22:36.978772 139606351599360 logging_writer.py:48] [56000] global_step=56000, grad_norm=0.5, loss=0.974049 +I0915 16:22:36.986040 139627431158976 submission.py:307] 56000) loss = 0.974, grad_norm = 0.500 +I0915 16:26:03.459869 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 16:26:19.263408 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 16:26:38.625142 139627431158976 spec.py:363] Evaluating on the test split. +I0915 16:26:48.927106 139627431158976 submission_runner.py:516] Time since start: 43180.85s, Step: 56306, {'train/ctc_loss': 0.09101925033906962, 'train/wer': 0.03626752768523868, 'validation/ctc_loss': 0.34150924820580925, 'validation/wer': 0.09240573552841211, 'validation/num_examples': 5348, 'test/ctc_loss': 0.1834717915268727, 'test/wer': 0.056202140840493166, 'test/num_examples': 2472, 'score': 41920.71384716034, 'total_duration': 43180.852402210236, 'accumulated_submission_time': 41920.71384716034, 'accumulated_eval_time': 1177.7626717090607, 'accumulated_logging_time': 4.720675945281982} +I0915 16:26:49.267649 139606351599360 logging_writer.py:48] [56306] accumulated_eval_time=1177.76, accumulated_logging_time=4.72068, accumulated_submission_time=41920.7, global_step=56306, preemption_count=0, score=41920.7, test/ctc_loss=0.183472, test/num_examples=2472, test/wer=0.0562021, total_duration=43180.9, train/ctc_loss=0.0910193, train/wer=0.0362675, validation/ctc_loss=0.341509, validation/num_examples=5348, validation/wer=0.0924057 +I0915 16:30:06.300826 139606343206656 logging_writer.py:48] [56500] global_step=56500, grad_norm=0.499999, loss=1.06638 +I0915 16:30:06.304993 139627431158976 submission.py:307] 56500) loss = 1.066, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 16:35:28.625240 139606351599360 logging_writer.py:48] [57000] global_step=57000, grad_norm=0.496036, loss=0.939441 +I0915 16:35:28.632780 139627431158976 submission.py:307] 57000) loss = 0.939, grad_norm = 0.496 +I0915 16:41:53.410023 139606343206656 logging_writer.py:48] [57500] global_step=57500, grad_norm=0.499999, loss=0.995549 +I0915 16:41:53.414198 139627431158976 submission.py:307] 57500) loss = 0.996, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 16:47:33.946289 139606351599360 logging_writer.py:48] [58000] global_step=58000, grad_norm=0.499999, loss=0.963871 +I0915 16:47:33.954149 139627431158976 submission.py:307] 58000) loss = 0.964, grad_norm = 0.500 +I0915 16:53:43.239849 139606343206656 logging_writer.py:48] [58500] global_step=58500, grad_norm=0.499999, loss=0.974066 +I0915 16:53:43.244037 139627431158976 submission.py:307] 58500) loss = 0.974, grad_norm = 0.500 +I0915 16:55:57.139644 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 16:56:13.840913 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 16:56:33.490050 139627431158976 spec.py:363] Evaluating on the test split. +I0915 16:56:43.703458 139627431158976 submission_runner.py:516] Time since start: 44975.63s, Step: 58623, {'train/ctc_loss': 0.08629992160600618, 'train/wer': 0.03433455538937513, 'validation/ctc_loss': 0.34272502142668776, 'validation/wer': 0.09178776613720852, 'validation/num_examples': 5348, 'test/ctc_loss': 0.18092379447867604, 'test/wer': 0.05402880181991754, 'test/num_examples': 2472, 'score': 43665.804336071014, 'total_duration': 44975.628787994385, 'accumulated_submission_time': 43665.804336071014, 'accumulated_eval_time': 1224.3262321949005, 'accumulated_logging_time': 5.071696996688843} +I0915 16:56:43.805537 139606351599360 logging_writer.py:48] [58623] accumulated_eval_time=1224.33, accumulated_logging_time=5.0717, accumulated_submission_time=43665.8, global_step=58623, preemption_count=0, score=43665.8, test/ctc_loss=0.180924, test/num_examples=2472, test/wer=0.0540288, total_duration=44975.6, train/ctc_loss=0.0862999, train/wer=0.0343346, validation/ctc_loss=0.342725, validation/num_examples=5348, validation/wer=0.0917878 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 17:00:27.977885 139606351599360 logging_writer.py:48] [59000] global_step=59000, grad_norm=0.499999, loss=0.981737 +I0915 17:00:27.985126 139627431158976 submission.py:307] 59000) loss = 0.982, grad_norm = 0.500 +I0915 17:06:28.957082 139606343206656 logging_writer.py:48] [59500] global_step=59500, grad_norm=0.499999, loss=0.958362 +I0915 17:06:28.961451 139627431158976 submission.py:307] 59500) loss = 0.958, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 17:12:44.526944 139606351599360 logging_writer.py:48] [60000] global_step=60000, grad_norm=0.499999, loss=0.989876 +I0915 17:12:44.534421 139627431158976 submission.py:307] 60000) loss = 0.990, grad_norm = 0.500 +I0915 17:18:16.875036 139606343206656 logging_writer.py:48] [60500] global_step=60500, grad_norm=0.499999, loss=0.937879 +I0915 17:18:16.879304 139627431158976 submission.py:307] 60500) loss = 0.938, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 17:24:56.712121 139606351599360 logging_writer.py:48] [61000] global_step=61000, grad_norm=0.499999, loss=0.972034 +I0915 17:24:56.719889 139627431158976 submission.py:307] 61000) loss = 0.972, grad_norm = 0.500 +I0915 17:25:52.941964 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 17:26:08.260189 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 17:26:27.959382 139627431158976 spec.py:363] Evaluating on the test split. +I0915 17:26:38.305024 139627431158976 submission_runner.py:516] Time since start: 46770.23s, Step: 61124, {'train/ctc_loss': 0.08153262054728035, 'train/wer': 0.03261755765170863, 'validation/ctc_loss': 0.3389452811055857, 'validation/wer': 0.0900980060831362, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17906773975337117, 'test/wer': 0.05327727337355026, 'test/num_examples': 2472, 'score': 45411.412904024124, 'total_duration': 46770.230377435684, 'accumulated_submission_time': 45411.412904024124, 'accumulated_eval_time': 1269.6891195774078, 'accumulated_logging_time': 5.183867931365967} +I0915 17:26:38.721827 139606351599360 logging_writer.py:48] [61124] accumulated_eval_time=1269.69, accumulated_logging_time=5.18387, accumulated_submission_time=45411.4, global_step=61124, preemption_count=0, score=45411.4, test/ctc_loss=0.179068, test/num_examples=2472, test/wer=0.0532773, total_duration=46770.2, train/ctc_loss=0.0815326, train/wer=0.0326176, validation/ctc_loss=0.338945, validation/num_examples=5348, validation/wer=0.090098 +I0915 17:31:10.525917 139606343206656 logging_writer.py:48] [61500] global_step=61500, grad_norm=0.499999, loss=0.95497 +I0915 17:31:10.530085 139627431158976 submission.py:307] 61500) loss = 0.955, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 17:38:04.808474 139606351599360 logging_writer.py:48] [62000] global_step=62000, grad_norm=0.499999, loss=0.933132 +I0915 17:38:04.815712 139627431158976 submission.py:307] 62000) loss = 0.933, grad_norm = 0.500 +I0915 17:43:04.238506 139606343206656 logging_writer.py:48] [62500] global_step=62500, grad_norm=0.499999, loss=0.960786 +I0915 17:43:04.242791 139627431158976 submission.py:307] 62500) loss = 0.961, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 17:50:15.988743 139606351599360 logging_writer.py:48] [63000] global_step=63000, grad_norm=0.499999, loss=0.921934 +I0915 17:50:15.996232 139627431158976 submission.py:307] 63000) loss = 0.922, grad_norm = 0.500 +I0915 17:55:00.928783 139606343206656 logging_writer.py:48] [63500] global_step=63500, grad_norm=0.499999, loss=0.966483 +I0915 17:55:00.933055 139627431158976 submission.py:307] 63500) loss = 0.966, grad_norm = 0.500 +I0915 17:55:47.389044 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 17:56:03.061361 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 17:56:23.427534 139627431158976 spec.py:363] Evaluating on the test split. +I0915 17:56:34.578031 139627431158976 submission_runner.py:516] Time since start: 48566.50s, Step: 63553, {'train/ctc_loss': 0.07727144007557617, 'train/wer': 0.030635991080250745, 'validation/ctc_loss': 0.335847412747074, 'validation/wer': 0.08865929609424034, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17931246320327118, 'test/wer': 0.05398817866065444, 'test/num_examples': 2472, 'score': 47156.53523993492, 'total_duration': 48566.50311303139, 'accumulated_submission_time': 47156.53523993492, 'accumulated_eval_time': 1316.8776497840881, 'accumulated_logging_time': 5.611002206802368} +I0915 17:56:34.914488 139606351599360 logging_writer.py:48] [63553] accumulated_eval_time=1316.88, accumulated_logging_time=5.611, accumulated_submission_time=47156.5, global_step=63553, preemption_count=0, score=47156.5, test/ctc_loss=0.179312, test/num_examples=2472, test/wer=0.0539882, total_duration=48566.5, train/ctc_loss=0.0772714, train/wer=0.030636, validation/ctc_loss=0.335847, validation/num_examples=5348, validation/wer=0.0886593 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 18:03:10.450338 139606351599360 logging_writer.py:48] [64000] global_step=64000, grad_norm=0.499999, loss=0.970424 +I0915 18:03:10.457722 139627431158976 submission.py:307] 64000) loss = 0.970, grad_norm = 0.500 +I0915 18:07:49.686460 139606343206656 logging_writer.py:48] [64500] global_step=64500, grad_norm=0.499999, loss=0.988419 +I0915 18:07:49.690555 139627431158976 submission.py:307] 64500) loss = 0.988, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 18:15:19.192241 139606351599360 logging_writer.py:48] [65000] global_step=65000, grad_norm=0.499999, loss=0.928571 +I0915 18:15:19.201293 139627431158976 submission.py:307] 65000) loss = 0.929, grad_norm = 0.500 +I0915 18:19:43.946550 139606343206656 logging_writer.py:48] [65500] global_step=65500, grad_norm=0.435909, loss=0.953693 +I0915 18:19:43.950841 139627431158976 submission.py:307] 65500) loss = 0.954, grad_norm = 0.436 +I0915 18:25:42.697845 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 18:25:58.356538 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 18:26:17.859890 139627431158976 spec.py:363] Evaluating on the test split. +I0915 18:26:28.106856 139627431158976 submission_runner.py:516] Time since start: 50360.03s, Step: 65857, {'train/ctc_loss': 0.07361976973495073, 'train/wer': 0.029259153271744588, 'validation/ctc_loss': 0.3315984976203034, 'validation/wer': 0.08708540530101869, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17693455021291696, 'test/wer': 0.05179452806044726, 'test/num_examples': 2472, 'score': 48901.461668252945, 'total_duration': 50360.032232522964, 'accumulated_submission_time': 48901.461668252945, 'accumulated_eval_time': 1362.2864887714386, 'accumulated_logging_time': 5.9613988399505615} +I0915 18:26:28.181034 139606351599360 logging_writer.py:48] [65857] accumulated_eval_time=1362.29, accumulated_logging_time=5.9614, accumulated_submission_time=48901.5, global_step=65857, preemption_count=0, score=48901.5, test/ctc_loss=0.176935, test/num_examples=2472, test/wer=0.0517945, total_duration=50360, train/ctc_loss=0.0736198, train/wer=0.0292592, validation/ctc_loss=0.331598, validation/num_examples=5348, validation/wer=0.0870854 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 18:28:13.685601 139606351599360 logging_writer.py:48] [66000] global_step=66000, grad_norm=0.499999, loss=0.880165 +I0915 18:28:13.692491 139627431158976 submission.py:307] 66000) loss = 0.880, grad_norm = 0.500 +I0915 18:32:37.422281 139606343206656 logging_writer.py:48] [66500] global_step=66500, grad_norm=0.499999, loss=0.907202 +I0915 18:32:37.426645 139627431158976 submission.py:307] 66500) loss = 0.907, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 18:40:31.610575 139606351599360 logging_writer.py:48] [67000] global_step=67000, grad_norm=0.499999, loss=0.977276 +I0915 18:40:31.619011 139627431158976 submission.py:307] 67000) loss = 0.977, grad_norm = 0.500 +I0915 18:44:36.204993 139606343206656 logging_writer.py:48] [67500] global_step=67500, grad_norm=0.499999, loss=0.925116 +I0915 18:44:36.209196 139627431158976 submission.py:307] 67500) loss = 0.925, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 18:52:44.855715 139606351599360 logging_writer.py:48] [68000] global_step=68000, grad_norm=0.499999, loss=0.906198 +I0915 18:52:44.873394 139627431158976 submission.py:307] 68000) loss = 0.906, grad_norm = 0.500 +I0915 18:55:36.830761 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 18:55:52.507913 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 18:56:13.116294 139627431158976 spec.py:363] Evaluating on the test split. +I0915 18:56:23.568814 139627431158976 submission_runner.py:516] Time since start: 52155.49s, Step: 68383, {'train/ctc_loss': 0.0712329912936901, 'train/wer': 0.028303465851722668, 'validation/ctc_loss': 0.33025378655879545, 'validation/wer': 0.08628397624680152, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17715023953158268, 'test/wer': 0.051449231206710945, 'test/num_examples': 2472, 'score': 50646.66412806511, 'total_duration': 52155.49419093132, 'accumulated_submission_time': 50646.66412806511, 'accumulated_eval_time': 1409.0245685577393, 'accumulated_logging_time': 6.045791149139404} +I0915 18:56:23.938944 139606351599360 logging_writer.py:48] [68383] accumulated_eval_time=1409.02, accumulated_logging_time=6.04579, accumulated_submission_time=50646.7, global_step=68383, preemption_count=0, score=50646.7, test/ctc_loss=0.17715, test/num_examples=2472, test/wer=0.0514492, total_duration=52155.5, train/ctc_loss=0.071233, train/wer=0.0283035, validation/ctc_loss=0.330254, validation/num_examples=5348, validation/wer=0.086284 +I0915 18:57:38.612292 139606343206656 logging_writer.py:48] [68500] global_step=68500, grad_norm=0.5, loss=0.942224 +I0915 18:57:38.616171 139627431158976 submission.py:307] 68500) loss = 0.942, grad_norm = 0.500 +I0915 19:05:44.026620 139606351599360 logging_writer.py:48] [69000] global_step=69000, grad_norm=0.478183, loss=0.934355 +I0915 19:05:44.030745 139627431158976 submission.py:307] 69000) loss = 0.934, grad_norm = 0.478 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 19:09:41.425627 139606351599360 logging_writer.py:48] [69500] global_step=69500, grad_norm=0.499999, loss=0.944248 +I0915 19:09:41.434629 139627431158976 submission.py:307] 69500) loss = 0.944, grad_norm = 0.500 +I0915 19:17:29.102805 139606343206656 logging_writer.py:48] [70000] global_step=70000, grad_norm=0.499999, loss=0.878192 +I0915 19:17:29.107008 139627431158976 submission.py:307] 70000) loss = 0.878, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 19:21:46.880197 139606351599360 logging_writer.py:48] [70500] global_step=70500, grad_norm=0.499999, loss=0.964381 +I0915 19:21:46.887533 139627431158976 submission.py:307] 70500) loss = 0.964, grad_norm = 0.500 +I0915 19:25:34.844704 139627431158976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 19:25:50.865714 139627431158976 spec.py:346] Evaluating on the validation split. +I0915 19:26:10.835541 139627431158976 spec.py:363] Evaluating on the test split. +I0915 19:26:21.170921 139627431158976 submission_runner.py:516] Time since start: 53953.10s, Step: 70799, {'train/ctc_loss': 0.06930567569555776, 'train/wer': 0.027574551717807642, 'validation/ctc_loss': 0.3291471334545158, 'validation/wer': 0.08531839907304592, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17562002488063747, 'test/wer': 0.050697702760343674, 'test/num_examples': 2472, 'score': 52393.79511928558, 'total_duration': 53953.09627389908, 'accumulated_submission_time': 52393.79511928558, 'accumulated_eval_time': 1455.3507027626038, 'accumulated_logging_time': 6.426264047622681} +I0915 19:26:21.485382 139606351599360 logging_writer.py:48] [70799] accumulated_eval_time=1455.35, accumulated_logging_time=6.42626, accumulated_submission_time=52393.8, global_step=70799, preemption_count=0, score=52393.8, test/ctc_loss=0.17562, test/num_examples=2472, test/wer=0.0506977, total_duration=53953.1, train/ctc_loss=0.0693057, train/wer=0.0275746, validation/ctc_loss=0.329147, validation/num_examples=5348, validation/wer=0.0853184 +I0915 19:26:22.277463 139606343206656 logging_writer.py:48] [70799] global_step=70799, preemption_count=0, score=52393.8 +I0915 19:26:22.460466 139627431158976 submission_runner.py:857] Final librispeech_conformer score: 52393.79511928558 +[W915 19:26:34.409239204 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator()) +[W915 19:26:34.409271314 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator()) +[W915 19:26:34.409269334 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator()) +[W915 19:26:34.409330065 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator()) diff --git a/logs/self_tuning/ademamix_golden/study_0/librispeech_conformer_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_0/librispeech_conformer_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..41c7e8fb --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/librispeech_conformer_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,32 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/ctc_loss,test/num_examples,test/wer,total_duration,train/ctc_loss,train/wer,validation/ctc_loss,validation/num_examples,validation/wer +65.22912073135376,0.0,24.23290634155273,1,0,24.23290634155273,30.430827795341635,2472,1.2926695509109742,90.46797966957092,31.778551183445963,1.6197443941103735,30.278628566405462,5348,1.2183266547578817 +113.29140305519104,0.0564334392547607,1770.718579530716,1472,0,1770.718579530716,4.081027808245693,2472,0.7480957894095424,1887.5654406547544,4.09133064106987,0.7841118316262344,4.29811154749347,5348,0.7659440930816396 +159.8078715801239,0.1082632541656494,3516.3809208869934,3276,0,3516.3809208869934,1.0242510605522936,2472,0.317084069628095,3682.5812780857086,1.0870527409677266,0.347314086400622,1.3146728859001406,5348,0.3709457828416936 +207.66377234458923,0.2054946422576904,5262.169579744339,5708,0,5262.169579744339,0.4453886641880121,2472,0.1497166534641399,5479.407527923584,0.4708599367690133,0.1614085860685611,0.6882235431422041,5348,0.2058417419012214 +255.93481731414795,0.2973103523254394,7009.676107883453,8062,0,7009.676107883453,0.3699733400138719,2472,0.1236772083764954,7278.852687597275,0.3739958180709729,0.1304216363312401,0.5951974787145871,5348,0.1773668710471684 +301.30780839920044,0.5694930553436279,8754.697697401047,10391,0,8754.697697401047,0.3373396317907607,2472,0.1113887026994089,9072.301375627518,0.3234601768658129,0.1144881133002532,0.5431776209626783,5348,0.1615314053975764 +347.27461290359497,0.6946163177490234,10500.55568909645,12921,0,10500.55568909645,0.306284679656752,2472,0.1031422013690004,10867.84413576126,0.2850455873176127,0.102955071892531,0.5090624882270947,5348,0.1518466663448076 +393.80775809288025,0.9752578735351562,12246.120055437088,15267,0,12246.120055437088,0.2837675515355829,2472,0.095972213759064,12663.76597571373,0.2590899828182181,0.0941919041936859,0.486629413269791,5348,0.1462752860522377 +437.9858956336975,1.0895891189575195,13991.609188079834,17623,0,13991.609188079834,0.280431247580489,2472,0.092153636788333,14456.398834943771,0.24585686104403,0.0900775888600322,0.4750910830445047,5348,0.1415439579008352 +485.1220736503601,1.382123947143555,15737.517180681229,20150,0,15737.517180681229,0.2655403171172333,2472,0.0872179229378668,16253.31299352646,0.2300486915802504,0.084246275788712,0.4586132341583785,5348,0.1357987737169893 +532.3313477039337,1.639185905456543,17482.665462493896,22490,0,17482.665462493896,0.2500807007710174,2472,0.082627505941137,18049.00102376938,0.2108367858610507,0.0783663684417975,0.4380481268522704,5348,0.1294935547723651 +576.6667482852936,1.731147527694702,19228.16298937797,24869,0,19228.16298937797,0.244341973675721,2472,0.0814900574817703,19841.71408224106,0.1999121621564489,0.0742466537441889,0.4275164290894364,5348,0.126123690435958 +622.7398328781128,2.002565860748291,20973.073004722595,27381,0,20973.073004722595,0.2384249084618362,2472,0.0782808278999857,21636.715457201004,0.1903591162585993,0.0709908372793684,0.4249022063994374,5348,0.1236421570994061 +667.5750579833984,2.254542827606201,22718.525512218475,29716,0,22718.525512218475,0.23190142811633,2472,0.0751731562163589,23430.025129795074,0.1791000074620681,0.0671032952318217,0.4094972616222121,5348,0.1179645633177231 +714.2352414131165,2.351791858673096,24463.878875732426,32106,0,24463.878875732426,0.2277881335086134,2472,0.0731216866735726,25224.99304747581,0.1720223388531498,0.0647437731835189,0.4075754574558594,5348,0.1166996572201033 +760.7859456539154,2.663914442062378,26208.83354616165,34613,0,26208.83354616165,0.2242102565084844,2472,0.0711311518696809,27020.623435735703,0.1605428295232393,0.0611315986976734,0.3960684971179927,5348,0.113204267851108 +807.6759321689606,3.0088658332824707,27953.99055099488,36937,0,27953.99055099488,0.212793970578747,2472,0.0683687770397903,28815.89855527877,0.1492908947759604,0.0571306700070731,0.3844368666176914,5348,0.1099502727755515 +855.6917436122894,3.0807156562805176,29699.269289255142,39346,0,29699.269289255142,0.212293056205239,2472,0.0658501411654784,30612.614881038666,0.1419760861854076,0.0547009562273564,0.3875349802450648,5348,0.1094385168734611 +904.376006603241,3.349537372589112,31445.309044122696,41839,0,31445.309044122696,0.2077698687213691,2472,0.0643267726931123,32411.072816848755,0.1351244860051342,0.0524386227302423,0.3787924452481414,5348,0.1071694105151354 +949.8861050605774,3.668992757797241,33191.16186285019,44160,0,33191.16186285019,0.200715570359378,2472,0.0631893242337456,34205.5338447094,0.1279378326222172,0.0490100266188643,0.3679224971980485,5348,0.1026987882006469 +995.6247532367706,3.748096227645874,34936.36906123161,46586,0,34936.36906123161,0.198968128992193,2472,0.0622956147299575,35999.97264456749,0.1209388803769168,0.0472066390579189,0.3711306384368093,5348,0.1029112151788731 +1041.9608659744265,4.022969007492065,36681.68979883194,49071,0,36681.68979883194,0.1923740165696174,2472,0.0600004062315926,37795.44325685501,0.1120992890530403,0.0439238257733239,0.3581757333735181,5348,0.0986819871578235 +1086.793828010559,4.299762964248657,38429.58753252029,51393,0,38429.58753252029,0.1892098255330989,2472,0.0579286251091747,39591.21010780335,0.1054451007400342,0.0414779139017423,0.3509819976579767,5348,0.0967218654950997 +1132.2957038879397,4.396579027175903,40174.67117357254,53851,0,40174.67117357254,0.1843811243870572,2472,0.0562630755793878,41385.38038396835,0.0978130419511976,0.0389078166591975,0.3491209956424553,5348,0.0955148940279051 +1177.762671709061,4.720675945281982,41920.71384716034,56306,0,41920.71384716034,0.1834717915268727,2472,0.0562021408404931,43180.85240221024,0.0910192503390696,0.0362675276852386,0.3415092482058092,5348,0.0924057355284121 +1224.3262321949005,5.071696996688843,43665.80433607101,58623,0,43665.80433607101,0.180923794478676,2472,0.0540288018199175,44975.62878799439,0.0862999216060061,0.0343345553893751,0.3427250214266877,5348,0.0917877661372085 +1269.6891195774078,5.183867931365967,45411.41290402413,61124,0,45411.41290402413,0.1790677397533711,2472,0.0532772733735502,46770.23037743568,0.0815326205472803,0.0326175576517086,0.3389452811055857,5348,0.0900980060831362 +1316.877649784088,5.611002206802368,47156.53523993492,63553,0,47156.53523993492,0.1793124632032711,2472,0.0539881786606544,48566.50311303139,0.0772714400755761,0.0306359910802507,0.335847412747074,5348,0.0886592960942403 +1362.2864887714386,5.9613988399505615,48901.46166825295,65857,0,48901.46166825295,0.1769345502129169,2472,0.0517945280604472,50360.03223252296,0.0736197697349507,0.0292591532717445,0.3315984976203034,5348,0.0870854053010186 +1409.0245685577393,6.045791149139404,50646.66412806511,68383,0,50646.66412806511,0.1771502395315826,2472,0.0514492312067109,52155.49419093132,0.0712329912936901,0.0283034658517226,0.3302537865587954,5348,0.0862839762468015 +1455.3507027626038,6.426264047622681,52393.79511928558,70799,0,52393.79511928558,0.17562002488063747,2472,0.050697702760343674,53953.09627389908,0.06930567569555776,0.027574551717807642,0.3291471334545158,5348,0.08531839907304592 diff --git a/logs/self_tuning/ademamix_golden/study_0/librispeech_conformer_pytorch/trial_1/events.out.tfevents.1789446428.bc3590e26d6c.38.0.v2 b/logs/self_tuning/ademamix_golden/study_0/librispeech_conformer_pytorch/trial_1/events.out.tfevents.1789446428.bc3590e26d6c.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..675f986f6ba4b0f94473aeb6af7993f7df342f72 GIT binary patch literal 84000 zcmbWAc~}qK8^$Z5vL##gLb6sOB1`sN_AOfyN+~TwQe=zDR`w-CT9BkrR9X>bi6pX= zQmM#N3BR|=Z|2-HXX=~&dauj-+}HV>=gjlmb7tmzhe}oby?CsjBmdujj@9;0AKs?X zQfnjA)f%JMnyj`Rq&>ZhLML03)i##a%grs1*Ze+3%) zs?3&Gari6HO`)qogB9kMOO4DI+t`{|5fV}$Vd&*)rTbUn?0UyJ@`17E7FLd5QMyM@ zh3*QCZ7b(Q)7aK{G0C!i(C^`()A;zJQ!o_c$E*%RQ>C-c#QOh3xjQ2nY1Ub9Rv7Yk zC`-nFE|AZC$x3J`XecyWYh-S^%*fW%QaGM7gTCN+ZmQ0%0Is5JR*c~y#E3zuemRDN z3Nh5n9{zVwQhs*=zZ&2XAEbYHCU0TmBkXMsC|Np$HDRfqlV`K<2 z?X~BSEMVh+qk4Z8+H?+FAm34mRnSoArO=d7pfWkPL8b$*r}%6Bm9kjaeyaS9cC1t< z1r-HZo64)_|B3&d5sfTiMT_r3uCmqG@TxUvY5U(gUS?;_B>(?9$BIJmdb-^9*w@j2 zhmrqbWsrO?4OT~2p}#^~BV%Jb3p;ZoTa#srZS0m>nA+IQ6k3p?|NXl;$Qjse!7-leyYSeEzjEtKR{P7_k7VU^6#6PClXB*k zD^{4UUIB`gv9O>L&b^P(`K$OWZs7;{@sn9a@iO3yi(%9L_!EQviYR&Iy_4TGhZX6q z&{ILt%G$)l!m9GxTWoA;x7zmqNe~Rj{*!`}#$Tnl`*X_WRp(4z^tYf0O5)|KGB`bT ze%EUVW#so_Hkw`(|NQwg6y~Fu<@p{kAL~}K5)#ixNO@TlP$+EPR7D{pEbKYgc!hmk z(|Ol`qLDirO{e(Hrzm{Wsy$GA_hKc)DWvpaD$-wkYYG&Z%EzlH>If+ue()$7O1GK{ z6p3BgXgbAzX~$(%*7&(BP%PADCB!N8k>b%=*X}^^_Ve{Bin>CI2A%8RiYhl6*9<7` zl(Ert3PCQu^f?g+6wjn@{C@!A6wXM|w*He3K=HcHvnml;A;sAk9)1Gnp#C6FQnLf><>m)bS!8M6rEz(XgY=9x@hM8D;OwdG z{C&D_fMQm^h$;#NA;n33o_W66c&95+m=9s2=@f!o+#l5xyI$mIRzjRY1u4!?>S72K z*ZyQxQ8W-zNcZ67!oNZ4aimbl{(rQD6xKaoDMBu8DzOsc6k148cCy(sph%e1ZV7XB zHxyDBFXB;@&XV>6itDr4XbCB-rmY_g6f#DvggAvhQsjkBM2g!!+o~uU2`SwAmSMEn zmZv*`V%jY>T0)9mb-x(_#hfHoLY!hMjw4&QeG%m1b-Ox7OfDJ=DY~5hiqT{HTzUf( z^?I<;5>m9VnBNB|6bG{s;uK489PLZ@zXA&PF#RfuCPE7TphAoenYZRHP&|)eqa~z} zG8ydw6j8~nggAv2j$>NdhjgHrch0(sLQzO@VG@sGeQf`UK(Q@|jh2vN)~W5cfMW73 zRzjS@87X=xZ=VGe$)zu=D4Gf>qWk7z^!8;kMnJJZI^+M*bP7SgD4aKP3Q*X~vl8MI zJ8>LpZ|@EUimARMmNM6cl91w7@*9laws&(XP#n9=M$;(-{i65li@2gzU1uf4DfZ$x zv~}_)0L9kjXR0Wg2`QYnrebt_-_@gmqSTU&rc+4s?=O@>en&tq&f2pQ;uME*9Nq(p zh5*HGx5z4r=0b{RarZE~k!l|^pm^uWMoUQH;CbddP|QEUN{CZ<<2e58txyJv#^Zx@7r?`OQh#ZoU0~GZXJ*q@n z3Mt}KuVJ*W+`&Z1MPFq$noc3OE-Y4GR{@FzDy)P!MG%fd&!g`TppbtpwT#I{D`-PU#3`b195N^M+X97sqa#%mt%Ve>YP>oktf*ZHY_xFGkCP@lp^(~Aq}EKs0lwEGhTo>v>is7o+NU=AnH@>RzsUaIJAw~G}9zTF$a0^yKoT434 z)W5o>B~avsZ>gf_CZw2k`yEoeuQN&pa&aw&jiyrwYLv~lxICb+OJXI&DO8Z6=&#VI!Yy8=0I)&iso^@mRT%ZWM%1Ve+bU_NI4}WmCdE2#N zX3UD}A*2`=$fNl5@F%*#)3?}YI)&is-lb)=5h$L;u@d4G-H>8M*CX2@7wRW+lWabde%F;ur4Zj9>Y+ib7LJF-`3yQoQNmo&gkkR%|q#La?Ha zw64JW3txLyLY!hSQpld%;0C$a5ZlU}Sy5U-3eT4>kRn<$?;uc&dB8@~DN6adnCO2C zJ?+3$RzjRYA1PwLnc{u-unr5WDEbH~9=&{y6k*?IVU2QEXQSy9f?V8fakLy()Hh95 zLY!h0Qp~Y%OoCj5C4Z`-&=ykY59Co~IAl9PF5Wz6qa~!+5Vs0<55~P=CB!MlA;nLH z7x?Z$+AVjPMOa=cTNKIK@moWdX%t9r2#;uJHHA}(fQ3Q)}HJa9F$ zqWTLd+$TLoiu-HY;!~JQJ=tgpDW(iRiF<=z`?3<^6myXx_Upd!K+$daMK815CO~18_o#|OS4iP@D;X)8 zo*wcYC{}!7qv;fa6}8;R!UZT^ma`J#6ibj|h{~i?posp^&XQSC1BDbX6nPX$iv4YX zqFE6eEg?m0UER|_vFHaYAx^OjDe@hsegO*4pSD#LgM<{13?5=GLK2Gi0EI^#yZ=Yi zDFiiY;Qa+{fns_?RzjR&B~ln{n)?*nTf46DS(JUrVYiQ!5U1FP6n8D} z830AQ5mr?c`a+6=n7c?}{O9H&ph%v;M$;+2@+o9~jadN{17@%i;uOwEvF@r9-mw>kMV%*MCn_4~0B;uLO3;p*{fG*En5P*g=RR!H$6H6AIN)aia5C_6`(i}-^P~7#W*2F&MY3q!=<0`i=Ly&Y&4zXC!Zol zJMT46ls#u9#3}Y7MZ=$2Q-Q)XZg>^Ncp-&aYaT`CR@**6E)G6mqv;facMmi}bp8Ov zrBqfzoMJyxq&GX_2o#U4y{jlD2r2Au$Ki@{-@f@IP)y&%M$;(@`MK~2knai<4%=A? zaSBhQ*!1dBAE0=;Wtttcq9zI{+OFnNgt}RE0*d3i*l0S1zzv@AQo^@_efP5x;uHsw zVn^v1d?IEvb$1oTBq7Dhc07vZ>r0*h#j<&9G@arzKNs5Fcj`bcJ{hqR;uMFG;?n4z znvjcF`3qGPlZ6xukMSsW&i;ve<}I7C(R2#IbG?oF@$o=0tSu`cPH_||Y!6T91{8Ml zepgXU5mKD<=TXR5boPc^7#gwBbPB=MeQ84|?&SEHvJ&DH$B`m7coI@{z7@8Xxh@QZ z6dh;8;?=#`tbbA|PRNDb16D$u;v`bcx7q0nxoEQKX%)p(A;ljb z9)zEZaT}V+;HwIVK^!kBuK+*d# z8%?JW?BqOI6VMMRcBZou;uJneq3a%=2P?|+r&T0fx<=0yozF$kYZ~dZ$%B zJdTwRr|?IL1;2v60mX-;%T*M!g%qJacksF}8h;1BI!t)VM$;(-&Rjk^=s8f_d&NqK zQv@P~>M5NQK+$v8y7kO;F-J%-UomxEg?ml?T)IDi?yp*32}-bq-Zs;`x>D5 zrFEr>VxExV#seNj{m`L!U8E0Wqv;faN~@|n$^j_MN3as&6v0T*rb`GqbN8!_H!v$| zzL3IU36COp=g9W3qS{8V(R2#IefQ>$9vMLKK9-dbrwBoc(OOw@kc-`!?KD1wx918$1fl zSq^`I;%p!rO{Wkz^C3}Z=R+>Gg|HIh6xWgBoIz?|ppdofS4FW^0wrx2{D&Uc+I0!1qqRzjTOCQ^8fE-C|x(y7K(6pMru+1fmcxqosR0Y%;UY&4xh z&@VCy_J#n3`%+dyoFW1#q+YDRo&OtabT=|9YO#=_cOxD}rf*t5plIO0M$;+s_;;+M zmw$W=6vJIu32}>MbM^1&W0*W4k*l0S1V9z{nw%Ku@ za2>%)h*Lx%MV^Q9d7#kwyrzo6NJ#M`mv^#@&FK$5K(V%rjiyrwuI?48(Jg@DfsFP4 z2Ov&y2Pp=%cE_iajn7`K5?LywxTeda`0;umcD)LJHkwW$SW)V2wxN%ja+Q@3r-((0 z;tMPAyQDfp4>>Z|g|U#rGnz*+;{1&Tkc;_a*l0S1z|*$0uv-qfh@Zkrh*QKPMf)N} z{6@^D;%XJeG9kr;6FiEy%auC<#rk?7|Bt3q2=-}vG#X_A6rUQi65l5Sq{dfOX`M0QVs|E6}4s+pj4na2FtAr2eUnL=B zFvqP(KJ*NyAr4YL5)LK1@cDiwfOwuB6vTk!Qchp0W4IhaS{(~s1|WL^xNtg1F~8i7 z9c$SDK%zo84RMeSqX6XHNtw)nf2U)2{Q;ll>mSPR(62HhQ4Oj)J5!@z3nI-4>v;r_ zZlB|X7$m{rnI7Te@T9fERONj`{|<&Z)iflC)2Qk6Ox0bkIHJr?#;$;te3MhF=J?Ey z%}3#|202(c!*Pd2iJZ_sou65@P%0V@i#+m_QxrcyGw~2_(vNO@ssPuu^&3uydV;2o z%P@qSWYO-$;Js!iV`O}y z6ABqClrP{dM(QD&%3tzP=;5mDnPdJA#9HUZIr&blI3Y<-(G(1C_MIi&^#7`MYxUJj zUa7bCTjk z;W|#AlX%W zK|b(7G}Z1F0?2p`PD2#LSr0*8=bUhYvR9ILIgCl6>NCmOkqxRsmxn~g0`aq*#=)t|^ zIEh;JV7(bj%%BISHQ}^`Jy<#35ErAd!HFx7v6rp5_-YQap4@#o3dQGrnFBp|K?hDq z(jF`r-oEcI6aAau*!{B*n>OAnAU-!P~4xB z6fFu-lW_W`+}>~vdhoj;oJ#F_u>6fh51~+MtkwC=)R#cYku^n4zTpB_TX$Iws>;^Q zT)2dVa$?1VaF~;8yEqN;Ihl`h^7`VyHvkf;7-iYNvUPb=&KTK!ItM`(8SZEXAf;`& za5_jXzv8R7ro8}=F>0KKD2UVs1o7~7`3xWpjHGTeAfA*%A<}xhMG%>$uT~<6B^OQy z$>M{kS}5iLNTLI$AqwK{h9C-OMl=SHKSNZy45-Y=PRdvOWf$em`I7XjqvcrKg{ zlEVkliJW#2KyJ+7G(0ER{NV5fRw)B!s#Fa zGx4W&W56YUmq8a0`TRhn8j$rlEeqaK-RM_$EffXYFiujmC@cxa zJm{s#^#?Pt?lz}VJ2SCaM)MC8N`t*kvzhu*OgT*;aO)$yhT9FXo(@%|*)cAhUMK~( zk1jfi+At@xd^io!IdKlhIT`IcWgUQQ-q?MRZe{DrqI~{+b#WqsnAy9O0?12OE}RaM z&2L?=bClx%GcH|GXjwB#H8YrOv+l2(oGTk4^w`|0EYq2YJiS z$XvN`$^i1(pVJTpS(1k!#_KmV1&}K(?M4r*Tv}0-FXF7XMIp$dalL{8WS1%zP6r8F z^b@;4l-9z}05ZM@ry&X=RgNG#<|<1;{k~|Bw3Gq)J+wMAVYD$4hqLVFnRiT6Oz(edvkwBC<?VSgZXfFpAdgpb z;dGGq{H4{>vw3|0>21$xh=S<1LlDihm!ME-6uphjdcvlO0@c0v)Mom39s8Iu)WF7sTL4Y>yJf~Jo4|ZJ1+h+(i z)IJ0~`0`aw=%0GvQ;;ZSwI?;_#XkA(7Jm06>N}(sW`#N-1BjWbF&a#AGY} zOb3ujt+;SHh@j#(UhpIhKzvm=4N;IcJrQJqt0}%Sk#e;~J_Axf`P|@j^eqHYUVTOd zKz2lN;dBte&O~CLn_2)eB8k%w1qtelAe%$GmqJFCzNpi8aAoUyLwP^^O57y`(OzGy z03i9a6>;|VTdn)xE665q9D$C2y)$4#uz|?(=E?1AWtbT^o}0jhah(H{p|qc zRyG$-2N7(s$bZXB29OD#ISo;esUs2O^{`&5P`{rWG|XZ^%9d4UCY(O+!@jn;{*JAH zG-d%O#2^XGM7#9I*w{vg2X6*5aef)6QIna-pC*HoJm#Ru7(m-?&Z$*nCayTPMYJt9 zQYM0#IJ=e;`X@7y{qby1FcVKVaf;$*V(z2kII=|%hwQeYIEVn9~piiJFffE#_Z84NFUFtXJEil^O9_BLoRv z?2jP#R`)0dkk|9La5~7LON%uCq{pB`h78DIJ0Xa# zbRdG5k4bm|AYP-ma5{+KEtZ!@5+VSkR;i*fSg@dotaqU zyA|g(e{KP8w#0;SLJX3?OkB>1!p8QsUSlON6Y5c%Mong7lX@AB=xyGWpMcgikyES2 zOlaM0fkN9ZOg;f-qWlpj^iO6&*3D!jn2A~GoT9jy_&J{E;MRZWi(fYBz2kJKW}>-! zEY7*HdIvc$6NBQ&aOqk@vqM5joGVnW?iEm9gJ&DYOv;TT5 zEX_SS-i4wtwGAgJS`^$?tvC z?z%>FI_?IAQa79nrx(h{k>Bw0Z7BWR8sab-&l0A$1gE}RbXioeC;)n4f^fNVU@X^4VIZ9ouumRx6KX;GFf z4=lS5%zVVKEL-pR-9)iPmNp1Vp{XDf1|9q{7yn!6|qsI-m~N(`02R~*@J9ednz730KVe}^;8 zWUZ0Biw_}EldX6OJ-LJ6gRvKE)FB(#Sf zy81RDDef&Ud+x!pEqHXo1}fI>ctVJ3E*geM;Ka}Oo_=b`--%x`{&Ru6!9zl$mhNI! z4`a1Hf5lF?xO|lNdPax|?S<7z-rdK5KaDaWZ`Q8~folB4;kQP(zFUQ!-Z=K}P;SQj zJSHEVO(;ofFa!f!{UxP*_+MEmzdHf)UkVA?njMDr?{st+t@hg99ragmGQyrlW<%+``(dXS zQ*Kh+D5W#njCWsoK+CWcip<6mBA#A48*Ti7*UQ`C+FxK^2K*vKMCZlZ4d>EW(Xy{T9Z+vN((o5oOuA7g?^f4|NBY zfk&@RWLU0yQdmZ>=2?vgv(0Y*kDh={USc_Pb&$TP;k z;yb+eEruoVFoorJ0&k5;?~ujc&r+XE#M4;{ExzL#YprcE09Y*M5F(;1a}Og+`j-br zz|v;d8krH5Ys~vNh2{DQo-5C}^=1j=<&_r^PiGNWz>s|VJYbPNO^ArH=pRRxrKTE_ zpr|)VS=p6g2|ZD*)p)mW6;7_wFcmzv?&eEEiJ=l$jj67I*zv~Y9JvTqBmF%gQj^s% zOwPkGJsN3w4d_1P6LQs9jnu{sP-%GBk2hd7V!jbd|711BTDd#|t8wEOAt`P(a=z}s zu@znlsRLFcLe`YaA=PRaoC?N?zZTFoAFRfz#)L*Kt%l=)dvm~Q>}^Si39W|G2RmGq z7usI@40+qso)D?!rL?dKBxSkM1%|>G(}5<`3aoDu%t_C!D@UTMab07YFL?X zX%D4Sbx)IxOu6y&rj*VLg98xhsCwE3D4ny967lrXnZ3UluNR+QX-8mQ?wlb+MCYZO zH_l5Z73)}FiD~#GmSNfHLtzP&KaDH}_XF{VEgy6s;^{2vCwMJwz_p46z*4M5h={T% z`XEakR~u|;n+uG4jjU{G6}}XfDbc>jvVWZw{?w>nDG^U+5xlj(`IYu)GC5B4Sp4~QiW0#meM}9xF z=b?s#NKNfIZR{HyQ<=5pRiGPaOvqK!p1&Qbi%QnhSE)jKo@zlT{Zo53H1M*8_B_ar zkQ8svo(j%5w$(H1Kc#hO&M?SZ)Ima^n)WO$$H%?6Jyjhe#bz2{&0y9l&C$YrSQH33~c+Z5K4q?D4{7@B0?_X6mFh*)2k* zcI|m@WSdb?I`vnL{>YS@tpCc5szOJFy5^QPHX{_|q?M#P)tz^j*9IyE+aHA@v&N2y zr7)B6!NIGI?YOu-vsKL_}GFu40aLG@H!_mY)w|PB1J;AtK6hI1E`rnib8u8aAVZ(DAG$tCT zMp_d}43)rYoNjy=JKn>z@?KyyPOc|JYO)%Z@6&Niv8mpvK(}`bAydNpY)@*lz=l%|CPe7uX3HeVh=YT8$A|=W*hv%WZrF zR^y2ep;1e#krB{o4Oop^frOaQYRu3*h^um%Y{wMHTgo*;pc<>Oy3R3F8vkOKH&~6B z2tr9xt05TR+}2L}!D?8?60$X0jldp?Xf?LGuRj1*!y}0hl*nofJ;*y`rt9o>{!k)Z zpAwp)C1S!IoWi0T$J>L|*zuB(sh!p6{cvYatpDeUuSoYqhusH5IiY&)h zE-M647~uCda`6c+ch-k6uj z>xXTCW$geWp3Wk8v3T#s9L&q+5rl{+OYvi5Y1b|OI~4VadLvdcEZzoq|8Fp*2sA%$nk>OUB5 z1Lf{<0HG;bBJxsk3d_DItAo`j3?^i1XElC|DOHEkS(v59?(CoulIt%;% z7zIV9ERBe#mrlXwgsGYivteG0-V!3B^OE)q=jB2013Zqbe^UHMhUHBSk_M-uULmUny$c*-ZUKd>}RBt%46F1$vTCEcFjPJmi)gg3*I_>RIdXz+1l zDeHCzf8kp{l8C3X2wtW1c8ePdET`@fBBCsN-yzGnqz$%^mp%>kau}9~4-}S-89WQP zG@v#9;_1DXL_D2EU^T*I+u*~XK`MlZD2x3EWN}h{@Cb_f{a$X}##gQ}&)jOQ#&44W z*zgX^-p_;8HdU8UVyFaG!`8wTt;T@2ZT-M%cnl*%YO)&MevfcWEr(l10NuW^gj_XN zW96aGIIxeM4>bg+)nU^SYqA;g4MqmknkT$L?S^NvH_6730rYOF@1hP>U> zoGvRZz-nA_B9tVx8iE08UX82(t8sh>AzQQ6c$n1{t;Uu}yH8*>H1-gJ5?PJa6BN;E zeAMq;4(0CcK|)isM0Cr=WMqB5tqE46*GWRAc2;AolEO$Ro#sQX8ZqT2D34M)ZB%)s zGk@xG+zB{6frzJ<&NWHD@Oru9Z{HH;rRi)!M08%3JR!ntX)XTqu}cp+r2LMev&2-9gbkfu(C4AtK5$t`J$KBshcu%lH-D+Dxcy1Bbs* zSnB$FBFpstWw(H($c~7ov)mlSJB&`d*T;Fla(gQwBFdum1z9Wtte*jk%Xd3NhQ;kW zg~hQBZzo{m;*wXu@>+T=7f)x&?wEpW%uXZc7O>QBK!}L4D1S$mZlNiL!18g)k4FrP zb2)|O`Q2lfm*>rt@#mL2SrhSe7Qq9@d68yQfMtjiAtK5mRgNrPTESsZ)TJ}#d}LVi z{$r^s>efc4t2K>njTe)TQN6or;pBRor6@s$)N%+VhDu;HmLA@U9dEVzBX_VG;YEZ< zO;#iPA}`?)!=*j~UGfh?t{SVMCsl+4yL{Er5v;~k>ABoI{gc&Dtr+zU9-!2bCnUwK z#>C;KIP>R<0`UP#j3OaKwHi;NypV6iksVvXYV>GLXw=ebT#S1>39Lqh3Lz%68h1MK z9_y`9-FXA@meh?9sK#nkG>yac-C|eg(_l3;wFo6it%hKLgLhnO4OSy=AR$|`)o8p& z8?DBqM(?|V)rc5Q2ufr%?zpr>s}ZHyEFVh5?D2%AXo+~rD-jpf<8;7kcuyr{YG*ZC zj@peMynVdcU1efrOZ)nZQaY9H^X{H6HlJn-Mdp1R5l=6joprwA^)mN-{t}p%)Q5zK z=)C0q!ljjeuA3UPf%TVqA7)r`H_Hg$JBi)TV^L2_#g=yICJ|3(5q!1b`!j9`ur!J% zM8sL7#=<$6&F)Db1D3^w*-sgkjBON_S26psrHvag7vH1*RYt_qS$s$EUVP7N*iH^u zBI9ic}(=<MG?YmJx z1+etYBt%46V(THxlUt9EKvAD(Fv6c)@_N=r%%#YBl;_L_EE83Lc;s zl=b=!^D;D<5D}f1>vA|RisOc?1eQ<5ttT-ofnF3AiwE1Wr5!X?o(3!?zleA`i{F+I zy!MxDvv`jza&5Rlh_VDKAj^PjSC#{dg~!E6hQ<2`g=O+Hp4Aw3ahW@?=o};B=`1=y z*~sE<`bG;_a?cSWqAcDGk;Nh55WWx8LMA|ZO63~!JV9akt?G_>=`=D6A37v7BI4;R zg54&qKBpsrrA=!>M3lv|F|vHVRL27vSPx5?RSe5r?`qp~Xr?kwZr`1OPNhYqsm2q9{FF0AW@eBDc?b%FLAbdk`g zWqa;&&-W{|=d0HUF=2cDY0P`v`f!ufPsrPoTZBM0?K!FATWrsUapii@o^xUeB}v<} zV1Nb}oVr4L?*4#~t+_pq$RCgG*<^pKNzk5+Qwc$d+VhBg`q-XFZT~bFN`%oXLQ}Lv zj90`GaZX-c1=@2_79mr+_UydVJQ3Q!uip6%Ou5;4no>Gjt#QTDxlDal4=6Ij1`zS| z(phpN39pwvpOo+zA1i%AM08$u^5!MznGJ5#|GYaihGD7jp|F&e>_wKzg;VhThK%P# zJe@_b6Cl&A9CrfjvI!AUmW@i7V^1yJ){vKAB`r;Z%9d7qfx^=9{yt0^C zh^Mm%?w(gZvG@b)b)PyRBFbXb99fF{1ziW0?Q1)2U|6yODJ(tmc{>5K%X?V?i;@cw zPiM*CzpK%-{pF3oBIiMfh_Wnei7YY+O=N+^$KByA!;%(6VR`$LmzUiwXFLFw@WVtr zokj4vZa42ocvMusGlYmJ%Uoq-@yZ|f0*d;MH-mdktz2WlSF5!e8cUnv1)S$I&m5=< z3kW5KN?JQ=O>7T4da-Zif!D^&5Atc4E#(ZO*)o}Jd(HN|Tbt^)MYBj7fwj*Cyh8jK|bC9)dUJw~I|sK5FMejr*r zh0qi&5y7poL}V7c*acSO&}>4cc2*R*@mUbdEHv)>x z_PInny>trh(r%g!z|#5Mh!7E-mq~4KUUo#AtOJ&b7YF(ukki|bfs~P0wfKIzv(<)oqg(wQk%YM8Q)>ch# z@C8`bjU(deEICpc$Z}J$UqfISVMvIGvUF2Hmgmu}2LQ|Y&^4DBmgBJ$7LPByH8%A6 zqr<>5Aclyiv!wOso#d=!w(%ga{CmQZ4-1XknS-PzdjG`8+i1FLan6CqNQ)oAfK8pqT)u}Bl>j_)Aks<9ep z9bV$V{umg30juG=k5Kw2s}YIzmP`v@T^ZZ*84XCh_DjTL{uYAiZK2vMy@^eq?U zJ5=w`d9WJo0|<>;T8+XDPIwl@(W``*&}y9Y|Aq$PTza2E$XiS}AyAFgQ0vco2;Jdr z{&27w*J21INv(!pfYt`ey}@d1zE8;3Y&8^>m!j3!X&`kOtj6M}grG!Lrl9FfXQU2oceFQS5^A5rVKuLF7b%|kdeo?K}=qytQ z8(ziWy1gJntBw##43)rY+`2suJ6?9UT^q0($)^dCnykj`B@sBL!cOt~pk=l9C*-QJ z8n9G-_!z?36y0z^=xbCxn>LYIOIN##OnxM+P2%?e~%psK#ozo{&YQ zk{?nH!D3q2+kJl+V?|-*mX|O^Arj%hTRCFM+#ba3`SrQHt)&%9fUx zNnx3(u>;$4r@p}rfTbXVh^MoB=5N$D_cGoEEiFHr5D{fb)I^qXe_r5E^$u`fc!pt# zcu!%O5Vr+cnl4^C30PhoCgSNVg7X_ZE$-q&hnHsw5mANot?;{6JHnG^Vq#aJe}nuKQB$&24_NE&L1H}L|KCRBFoy5E4~BEr{3S@&8l2u zKKT@u^shV%m@(DfAM&zfFcD8@5uC6Y;FEF@SeA|_L_}G9bdcqldx9A><4J9rTQV%= z|FKjRb?R3e4=??|W);>cO$w+CyAnzamB4C584gF0rBb0U!D@tR5+XHOjc0XlV^L{v zBPI&yEm7Eh02(X*FD$75xIMQMQ5*6IzW$_Pn>$K6XjKZ;9?% z5dzg%jj~g`_o@ya-8KxYhKD_&B&pR94A8kp%avd?+_n<3HCv5}9DB4H1sA5i2dhzc z4)p8*>IUD zH;0QTrL%Al@7Z&xZGjUMnO|RtczWp+eDJoxz57=von8MBBBJwhcmS5pd#Bok1B=H4 zLzUT;ZNROB!m`SeclSKhRxTD;GF}kzbe4R6OKUdjObW2%WDz2wEN%mlC9BKl(ZG^e zaBV5W;{1cc;!?nCX{sY5f`Mi74#2k`SU=jfy*V$k(#ap*L8K%=3gsEv?4tdI~{cH3BXXVnVBNWqcD{l?zPE214Fk zLkWRutj3zN%}{B3ng2ns8n|g#A6Shq z4+%kutVWLrXS5n$1GP<|M09*kXo{AI%e)d{y40)!tcKBRLZ)_BLo*@uJ+y(mh5HXN z3NsOISK}yZHTrc}ce=L`3Ih!Ejt!+d4&l1(t8U zKRjbtQnpiAZe8YmwV^w!yaTYb7)Zp^Sp@HDH1chi4lIv`6C$E4Q%54pr1#nQ6J?E+ zvL?^1Y-zE(C@l8Myu3`0`!XI_(mN3GbQZzg^Olw__zq3mo`i@f%c#-Fa#!{|9_Fn* zP<0o>a@~W%GNT30YBasq3Xkx5JCcZ}vk0ultXt=dATLUj2@z2i-Lc5>^R0AuD1>f5 zo)3kf~jJUKi+s&(U+H zS2Uhi*#-jlQ%a{+Pe*LeBTiYBK^v&Qgovk?PQhUIw>Z z=m_mOZq4(F42$?e$SXv+9?a;{79jpy3|9vIL#nV{?@0~o|n7$BLyp|CnqAYR-$a3mWrv!vO>Aj z1t{v%manGhhc;jl~-Yk(#W= zugG9*!lTaL9}jfjwi0sHSdFPCVsK!AVHTQTHC}rVO8;av9(8&Y23Di@Awp8zYOH)X z7%5+j?w0^oWArIPh-x+3*)B)E+2>xZ1FNCuOK8;6YRI^i+kw^4zD$S-twvxW?@j-5AHA1I4bcPb4ltyTZmI%Y?IE8w5_J0Sf(I%6Ssh!n$+D3L4 zl+H)`f8v;OQ+$q6I{UBWJ$oM2u~Y$yOs@$p>{|Es1H+PZg~Bqya6PtXEB~M2z;fS(h^Mm%-fx(=aXG%O+rga>5oJl5i!6VJ z*x@@gp`}CnE~s2%QP(IeWBDxpGa57mmOpidaq)B(!8f@| ze-*JCtVY2$LQ>pn$i2}+O4&wRUV_!gcPE6XR%1Xvp4B+k#-j|ZhQUEXqn1{quVE)Xr+SpZoFz+CcD%1813X6LN!6I%S-AR>QM-8*?a~b8U!tdg&Cj=QZv| z!(m=@HWDJD^AfTE=Ox%z-Ue8@8+QB3uw1xBVd>{&g)MFAorfjB@?-@OPiI+C$~(;a zeYGmdbP6eycu8@!vw;C-+4MfTo2KlGJYK;F*2vM!Zl*tQ`Z)4YBJ+K-U zzX^?6T8)7(ofd=D_*3sIx2S|xDp z%cgcjJe@`GDy8YLYy+@O^}7)wqAW|ykR@@-am>r5?z(FkmZxbHmRHI=3)pna^iII? zwI2~rXA!)s@gS@}?rOXqLWqd6%rr+9&D^pUke9cgdZ#lii5V1@e$VZ3jaf*CEC-g{ zG9sSNB6#3v;AnLl@={V)gBye>%ed9ZGHUgMwoueNueemgu*ANq)@mFmKab5y=~BT# zpc-aPC^1w5tMTiIEQ;J6^ZPbfjWG^{NKIB_@UDy4gq2Pt`T^Y$7ecNYt1)HUbsSjz z8w+t)V~abX^iNhJ%~by&SdAB6grvCDa9zmzLAC346Y&6S%M*kU)oP3$I3M|JFZyi( zt8x1rp;1e#;dtiM9k3d$FA`!xt6^r$`^GDE{qq>eTWSa)P>t2l3+|8Wdr<16dSEr~ zhZ9PYS`EPfb)?5F0jpsYN66M}H5!GUM5|GxId3{xjh{(`phQ+<*t>IRHDurJ?F1!a zQ7WM+S|Vb3C89*_V;`^@^D+pT+F1>i^gR3!@pAI%0ZS@dTErVl>HJ`Ci>0%5cycim z8HaQto?bcy4^ZMy-i&~G*^@`BFfUm7Fh=M%6<(jD^C8IWmLJw4i`{ZGU}LPUgS!3@u#r! z{fT%wi{KrahI(<0V2$+&B}7D7l-DB5jh%IHx9Q6fy%P+JTQP;jdeu5)al3Zk0rHaT zL&Vcr1nf^5Nr^N9t8uS_kQBEXK__@tLr#0$G}zTRT0fl2A=PSJ>%`mD z2pbym60C-FGeV=5R%1x{r?t@mtYD4FqU^P7E5rPs~jhXIO(Q4Rl zeP0IU&S52?DOw_i^GbwiNLUwqO@{fn3uCd2@%nGk=lTZw5*A*F|c%KVZM-I`TB#xA|ub+ z3CM5n=m#v1RET&9Sq{1N-wP~W-3bvs~WXSRfp~O%L+H+P=5w_>p;0vnIo|k42A~m(=u?c=SNl{P6H3B-hY(lP@_MAE| z80lW7_V&c~Ttq1SQ+s~$_*64!&xd~ylH%=omWB>e9@I8m1ns%v4|rSxn?2vpOa<*tvy^=)9^ z)(YD5%npQ-r0rQSK=(+ksnDLUb|qwMZqGf(`(t~aV>sUr+OxVQAt+IMPK>#Y?RnFR z;fIy%qTsPea z+kjKOaD2ZZeGm~(FP(xr&fsOCQ(;~bM-w7q^CFeYo0k-YX)VJ?3`^_|3d@S2 zE0M+9;IRi3%q4n6yo4+=x6H2rOZg~5M3m*-X3VkeuuxrKS-&=C?6S(1c6~R6C0b(x zvZ(nOeg_tXtwcPXMX*udbq3e3xa z#wYgx%b}S>Je@`G&>_8tCqBH)UP_3FvZQQ97VDOQ`p}H`PO8u`sa#{H4pwV5j=tml zZey;R^%bD{IhRmks03CcFe3-8#)n8R1+W^Imk=T~S&e$FcvfRTaJ?NsH_McetHx^h zUgN#J@%uvhJ+KSF>TyrNhYH2mTo}KLjR>RSY5EEJr|7W~Uu)p2-ik~f~ogf6Nu^LX{yf5E2 z?^$&btj1~|LP=7qAsFDRu-unmHSz-p*_y4!U6mlT8f9&C9)i_q5<&<{WHmaji$<%F zwamR9l)L0`LQ||nIPckpCF1qzW+TCB^o=28YG*Z~&)&ed(Br53j$_J==OIezJfy?> zsB&LJr8`h$zAPZ(>7}!T|7uuiztH6{FW*cE5wUrZy2qOrx0bj1K}$QGE`5h#*?Ek@ zlG(=++rSU$MvH;vP&N@yXIc1^_haQlo0K00mbZn3h$u_!cFb}1%+3>mMZ@3l7sFC< zlEO0dCND4h->awqi*pzePiGOFN;&b}Tzn_MG=>lnWx2frS?n!~Qh{aptufZiD_dIe z848Qmqa~P^V$T~Xz+#a=#M4;>4;{4K_L@LmwxtjvqAb^UBFh`U1s8$kmjCr&h9&Df zg~k6Z@3m0NdG&Syi)||6q^=LvwlqJ{|S)A%Mx(>~_+o;7q7?wSL z)mn{C#qQXwzHg241S-90gc3s~up0HV-lEm`)c?8^SdF-Ogh)+Rqiov+yx(_qNx(-c zX-0%xHCDqc{1VbFpRSw_R^$09Lg}BZM$;vSzJS$uY(q$jTaAd+zDQYq>oJ~6xy^wP zqFN1;3kJy7aID87uo|H*ghnl`#(cvK9l>fm+)aoHtwxgHBwUq)=6&4{t8#=VAyAFg zXdTV_iq9}its7Vk*JFf|q*g;Pz;^3hzJb+Pcb1T?*=h{TzlK($cuH>{uo|xe2tkRg z#@DkC&}!(&&7J`z;>A@$Q>;WdkKKjI(7CxBf8e`wI3ZIzt1)TN^)0Y6lGFEb&1 | tee -a /logs/librispeech_deepspeech_pytorch_09-16-2026-01-52-47.log +W0916 01:52:49.353000 9 site-packages/torch/distributed/run.py:803] +W0916 01:52:49.353000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0916 01:52:49.353000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0916 01:52:49.353000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-16 01:52:51.169740: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 01:52:51.169740: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 01:52:51.169740: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 01:52:51.169759: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789523571.190567 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789523571.190567 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789523571.190566 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789523571.190582 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789523571.197399 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789523571.197400 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789523571.197401 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789523571.197450 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789523571.214710 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789523571.214724 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789523571.214725 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789523571.214746 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789523571.214746 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789523571.214748 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789523571.214742 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789523571.214749 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789523571.214750 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789523571.214751 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789523571.214751 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789523571.214753 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789523571.214759 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789523571.214779 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789523571.214781 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789523571.214783 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789523576.994294 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789523577.014395 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789523577.090276 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789523577.146401 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W916 01:52:59.501946518 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0916 01:53:00.157240 139747222361280 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/librispeech_deepspeech_pytorch. +I0916 01:53:00.157239 139672845759680 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/librispeech_deepspeech_pytorch. +I0916 01:53:00.157238 140207082284224 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/librispeech_deepspeech_pytorch. +I0916 01:53:00.157282 139679763571904 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/librispeech_deepspeech_pytorch. +I0916 01:53:00.186713 139747222361280 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/librispeech_deepspeech_pytorch/trial_1. +I0916 01:53:00.431620 139747222361280 submission_runner.py:242] Initializing dataset. +I0916 01:53:00.431803 139747222361280 input_pipeline.py:19] Loading split = train-clean-100 +I0916 01:53:00.502717 139747222361280 input_pipeline.py:19] Loading split = train-clean-360 +I0916 01:53:00.694869 139747222361280 input_pipeline.py:19] Loading split = train-other-500 +I0916 01:53:01.199710 139747222361280 submission_runner.py:251] Initializing model. +W0916 01:53:04.853189 140207082284224 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0916 01:53:04.853304 139747222361280 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0916 01:53:04.854041 139679763571904 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0916 01:53:04.854135 139672845759680 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +I0916 01:53:06.547987 139747222361280 submission_runner.py:294] Initializing optimizer. +I0916 01:53:06.548539 139747222361280 submission_runner.py:299] Initializing metrics bundle. +I0916 01:53:06.548681 139747222361280 submission_runner.py:321] Initializing checkpoint and logger. +I0916 01:53:06.550068 139747222361280 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_0/librispeech_deepspeech_pytorch/trial_1/meta_data_0.json. +I0916 01:53:06.550177 140207082284224 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 01:53:06.550201 139679763571904 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 01:53:06.550312 139747222361280 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 01:53:06.550323 140207082284224 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 01:53:06.550337 139679763571904 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 01:53:06.550379 139747222361280 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 01:53:06.550423 139672845759680 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 01:53:06.550568 139672845759680 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 01:53:06.829031 139747222361280 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_0/librispeech_deepspeech_pytorch/trial_1/flags_0.json. +I0916 01:53:06.841214 139747222361280 submission_runner.py:359] Starting training loop. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 01:53:41.412780 139728833734400 logging_writer.py:48] [0] global_step=0, grad_norm=0.5, loss=33.7517 +I0916 01:53:41.452341 139747222361280 submission.py:307] 0) loss = 33.752, grad_norm = 0.500 +I0916 01:53:42.018182 139747222361280 spec.py:333] Evaluating on the training split. +I0916 01:53:42.019132 139747222361280 input_pipeline.py:19] Loading split = train-clean-100 +I0916 01:53:42.043181 139747222361280 input_pipeline.py:19] Loading split = train-clean-360 +I0916 01:53:42.131925 139747222361280 input_pipeline.py:19] Loading split = train-other-500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 01:54:16.461062 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 01:54:16.462126 139747222361280 input_pipeline.py:19] Loading split = dev-clean +I0916 01:54:16.478610 139747222361280 input_pipeline.py:19] Loading split = dev-other +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 01:54:42.465854 139747222361280 spec.py:363] Evaluating on the test split. +I0916 01:54:42.466909 139747222361280 input_pipeline.py:19] Loading split = test-clean +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 01:54:54.480597 139747222361280 submission_runner.py:516] Time since start: 107.64s, Step: 1, {'train/ctc_loss': 31.939386414583453, 'train/wer': 2.4035197577988447, 'validation/ctc_loss': 30.99260347598955, 'validation/wer': 2.2569111186211557, 'validation/num_examples': 5348, 'test/ctc_loss': 31.056942383379575, 'test/wer': 2.3705238356386977, 'test/num_examples': 2472, 'score': 34.61258101463318, 'total_duration': 107.63924741744995, 'accumulated_submission_time': 34.61258101463318, 'accumulated_eval_time': 72.46234583854675, 'accumulated_logging_time': 0} +I0916 01:54:54.507227 139728305223424 logging_writer.py:48] [1] accumulated_eval_time=72.4623, accumulated_logging_time=0, accumulated_submission_time=34.6126, global_step=1, preemption_count=0, score=34.6126, test/ctc_loss=31.0569, test/num_examples=2472, test/wer=2.37052, total_duration=107.639, train/ctc_loss=31.9394, train/wer=2.40352, validation/ctc_loss=30.9926, validation/num_examples=5348, validation/wer=2.25691 +I0916 01:54:55.974863 139728296830720 logging_writer.py:48] [1] global_step=1, grad_norm=0.5, loss=33.1824 +I0916 01:54:55.977781 139747222361280 submission.py:307] 1) loss = 33.182, grad_norm = 0.500 +I0916 01:54:56.732311 139728305223424 logging_writer.py:48] [2] global_step=2, grad_norm=0.5, loss=33.6702 +I0916 01:54:56.735448 139747222361280 submission.py:307] 2) loss = 33.670, grad_norm = 0.500 +I0916 01:54:57.279974 139728296830720 logging_writer.py:48] [3] global_step=3, grad_norm=0.5, loss=33.6519 +I0916 01:54:57.282924 139747222361280 submission.py:307] 3) loss = 33.652, grad_norm = 0.500 +I0916 01:54:57.825584 139728305223424 logging_writer.py:48] [4] global_step=4, grad_norm=0.5, loss=33.2787 +I0916 01:54:57.828515 139747222361280 submission.py:307] 4) loss = 33.279, grad_norm = 0.500 +I0916 01:54:58.365957 139728296830720 logging_writer.py:48] [5] global_step=5, grad_norm=0.5, loss=33.4655 +I0916 01:54:58.368865 139747222361280 submission.py:307] 5) loss = 33.465, grad_norm = 0.500 +I0916 01:54:58.917675 139728305223424 logging_writer.py:48] [6] global_step=6, grad_norm=0.5, loss=33.5473 +I0916 01:54:58.920637 139747222361280 submission.py:307] 6) loss = 33.547, grad_norm = 0.500 +I0916 01:54:59.457921 139728296830720 logging_writer.py:48] [7] global_step=7, grad_norm=0.5, loss=32.4647 +I0916 01:54:59.460813 139747222361280 submission.py:307] 7) loss = 32.465, grad_norm = 0.500 +I0916 01:55:00.001189 139728305223424 logging_writer.py:48] [8] global_step=8, grad_norm=0.5, loss=32.8299 +I0916 01:55:00.004028 139747222361280 submission.py:307] 8) loss = 32.830, grad_norm = 0.500 +I0916 01:55:00.957908 139728296830720 logging_writer.py:48] [9] global_step=9, grad_norm=0.5, loss=32.4651 +I0916 01:55:00.960939 139747222361280 submission.py:307] 9) loss = 32.465, grad_norm = 0.500 +I0916 01:55:01.822880 139728305223424 logging_writer.py:48] [10] global_step=10, grad_norm=0.5, loss=32.148 +I0916 01:55:01.825892 139747222361280 submission.py:307] 10) loss = 32.148, grad_norm = 0.500 +I0916 01:55:02.875445 139728296830720 logging_writer.py:48] [11] global_step=11, grad_norm=0.5, loss=32.2062 +I0916 01:55:02.878369 139747222361280 submission.py:307] 11) loss = 32.206, grad_norm = 0.500 +I0916 01:55:03.724021 139728305223424 logging_writer.py:48] [12] global_step=12, grad_norm=0.5, loss=32.203 +I0916 01:55:03.727158 139747222361280 submission.py:307] 12) loss = 32.203, grad_norm = 0.500 +I0916 01:55:06.486467 139728296830720 logging_writer.py:48] [13] global_step=13, grad_norm=0.5, loss=31.426 +I0916 01:55:06.489507 139747222361280 submission.py:307] 13) loss = 31.426, grad_norm = 0.500 +I0916 01:55:07.445056 139728305223424 logging_writer.py:48] [14] global_step=14, grad_norm=0.5, loss=31.1689 +I0916 01:55:07.448021 139747222361280 submission.py:307] 14) loss = 31.169, grad_norm = 0.500 +I0916 01:55:08.179293 139728296830720 logging_writer.py:48] [15] global_step=15, grad_norm=0.5, loss=29.9022 +I0916 01:55:08.182301 139747222361280 submission.py:307] 15) loss = 29.902, grad_norm = 0.500 +I0916 01:55:09.302706 139728305223424 logging_writer.py:48] [16] global_step=16, grad_norm=0.5, loss=30.2691 +I0916 01:55:09.305614 139747222361280 submission.py:307] 16) loss = 30.269, grad_norm = 0.500 +I0916 01:55:12.021591 139728296830720 logging_writer.py:48] [17] global_step=17, grad_norm=0.5, loss=29.5108 +I0916 01:55:12.024508 139747222361280 submission.py:307] 17) loss = 29.511, grad_norm = 0.500 +I0916 01:55:13.304939 139728305223424 logging_writer.py:48] [18] global_step=18, grad_norm=0.5, loss=29.1966 +I0916 01:55:13.307972 139747222361280 submission.py:307] 18) loss = 29.197, grad_norm = 0.500 +I0916 01:55:13.855706 139728296830720 logging_writer.py:48] [19] global_step=19, grad_norm=0.5, loss=28.1756 +I0916 01:55:13.858661 139747222361280 submission.py:307] 19) loss = 28.176, grad_norm = 0.500 +I0916 01:55:15.287512 139728305223424 logging_writer.py:48] [20] global_step=20, grad_norm=0.5, loss=27.4894 +I0916 01:55:15.290372 139747222361280 submission.py:307] 20) loss = 27.489, grad_norm = 0.500 +I0916 01:55:17.424204 139728296830720 logging_writer.py:48] [21] global_step=21, grad_norm=0.5, loss=27.0666 +I0916 01:55:17.427170 139747222361280 submission.py:307] 21) loss = 27.067, grad_norm = 0.500 +I0916 01:55:18.726447 139728305223424 logging_writer.py:48] [22] global_step=22, grad_norm=0.5, loss=27.1526 +I0916 01:55:18.729314 139747222361280 submission.py:307] 22) loss = 27.153, grad_norm = 0.500 +I0916 01:55:19.270893 139728296830720 logging_writer.py:48] [23] global_step=23, grad_norm=0.5, loss=26.044 +I0916 01:55:19.273806 139747222361280 submission.py:307] 23) loss = 26.044, grad_norm = 0.500 +I0916 01:55:20.963901 139728305223424 logging_writer.py:48] [24] global_step=24, grad_norm=0.5, loss=25.9057 +I0916 01:55:20.966793 139747222361280 submission.py:307] 24) loss = 25.906, grad_norm = 0.500 +I0916 01:55:23.304931 139728296830720 logging_writer.py:48] [25] global_step=25, grad_norm=0.5, loss=25.456 +I0916 01:55:23.307972 139747222361280 submission.py:307] 25) loss = 25.456, grad_norm = 0.500 +I0916 01:55:24.307175 139728305223424 logging_writer.py:48] [26] global_step=26, grad_norm=0.5, loss=24.5125 +I0916 01:55:24.310198 139747222361280 submission.py:307] 26) loss = 24.513, grad_norm = 0.500 +I0916 01:55:24.962636 139728296830720 logging_writer.py:48] [27] global_step=27, grad_norm=0.5, loss=24.0844 +I0916 01:55:24.965720 139747222361280 submission.py:307] 27) loss = 24.084, grad_norm = 0.500 +I0916 01:55:26.382540 139728305223424 logging_writer.py:48] [28] global_step=28, grad_norm=0.5, loss=23.1254 +I0916 01:55:26.385423 139747222361280 submission.py:307] 28) loss = 23.125, grad_norm = 0.500 +I0916 01:55:29.065450 139728296830720 logging_writer.py:48] [29] global_step=29, grad_norm=0.5, loss=22.477 +I0916 01:55:29.068350 139747222361280 submission.py:307] 29) loss = 22.477, grad_norm = 0.500 +I0916 01:55:29.842408 139728305223424 logging_writer.py:48] [30] global_step=30, grad_norm=0.5, loss=21.7468 +I0916 01:55:29.845389 139747222361280 submission.py:307] 30) loss = 21.747, grad_norm = 0.500 +I0916 01:55:30.397795 139728296830720 logging_writer.py:48] [31] global_step=31, grad_norm=0.5, loss=21.0763 +I0916 01:55:30.400650 139747222361280 submission.py:307] 31) loss = 21.076, grad_norm = 0.500 +I0916 01:55:31.877656 139728305223424 logging_writer.py:48] [32] global_step=32, grad_norm=0.5, loss=20.6661 +I0916 01:55:31.880570 139747222361280 submission.py:307] 32) loss = 20.666, grad_norm = 0.500 +I0916 01:55:34.507032 139728296830720 logging_writer.py:48] [33] global_step=33, grad_norm=0.5, loss=20.1122 +I0916 01:55:34.510060 139747222361280 submission.py:307] 33) loss = 20.112, grad_norm = 0.500 +I0916 01:55:35.830526 139728305223424 logging_writer.py:48] [34] global_step=34, grad_norm=0.5, loss=19.392 +I0916 01:55:35.833416 139747222361280 submission.py:307] 34) loss = 19.392, grad_norm = 0.500 +I0916 01:55:36.373312 139728296830720 logging_writer.py:48] [35] global_step=35, grad_norm=0.5, loss=18.8706 +I0916 01:55:36.376203 139747222361280 submission.py:307] 35) loss = 18.871, grad_norm = 0.500 +I0916 01:55:37.293299 139728305223424 logging_writer.py:48] [36] global_step=36, grad_norm=0.5, loss=18.1257 +I0916 01:55:37.296178 139747222361280 submission.py:307] 36) loss = 18.126, grad_norm = 0.500 +I0916 01:55:39.773358 139728296830720 logging_writer.py:48] [37] global_step=37, grad_norm=0.5, loss=16.817 +I0916 01:55:39.776326 139747222361280 submission.py:307] 37) loss = 16.817, grad_norm = 0.500 +I0916 01:55:41.437012 139728305223424 logging_writer.py:48] [38] global_step=38, grad_norm=0.5, loss=16.8527 +I0916 01:55:41.439962 139747222361280 submission.py:307] 38) loss = 16.853, grad_norm = 0.500 +I0916 01:55:41.982667 139728296830720 logging_writer.py:48] [39] global_step=39, grad_norm=0.5, loss=15.881 +I0916 01:55:41.985535 139747222361280 submission.py:307] 39) loss = 15.881, grad_norm = 0.500 +I0916 01:55:42.830468 139728305223424 logging_writer.py:48] [40] global_step=40, grad_norm=0.5, loss=15.6715 +I0916 01:55:42.833394 139747222361280 submission.py:307] 40) loss = 15.671, grad_norm = 0.500 +I0916 01:55:45.544269 139728296830720 logging_writer.py:48] [41] global_step=41, grad_norm=0.5, loss=14.8486 +I0916 01:55:45.547250 139747222361280 submission.py:307] 41) loss = 14.849, grad_norm = 0.500 +I0916 01:55:46.996743 139728305223424 logging_writer.py:48] [42] global_step=42, grad_norm=0.5, loss=14.4225 +I0916 01:55:46.999646 139747222361280 submission.py:307] 42) loss = 14.423, grad_norm = 0.500 +I0916 01:55:47.547980 139728296830720 logging_writer.py:48] [43] global_step=43, grad_norm=0.5, loss=13.9482 +I0916 01:55:47.551043 139747222361280 submission.py:307] 43) loss = 13.948, grad_norm = 0.500 +I0916 01:55:48.268670 139728305223424 logging_writer.py:48] [44] global_step=44, grad_norm=0.5, loss=13.0447 +I0916 01:55:48.271641 139747222361280 submission.py:307] 44) loss = 13.045, grad_norm = 0.500 +I0916 01:55:50.915625 139728296830720 logging_writer.py:48] [45] global_step=45, grad_norm=0.5, loss=12.3351 +I0916 01:55:50.918554 139747222361280 submission.py:307] 45) loss = 12.335, grad_norm = 0.500 +I0916 01:55:52.491232 139728305223424 logging_writer.py:48] [46] global_step=46, grad_norm=0.5, loss=11.812 +I0916 01:55:52.494110 139747222361280 submission.py:307] 46) loss = 11.812, grad_norm = 0.500 +I0916 01:55:53.033860 139728296830720 logging_writer.py:48] [47] global_step=47, grad_norm=0.5, loss=11.5711 +I0916 01:55:53.036716 139747222361280 submission.py:307] 47) loss = 11.571, grad_norm = 0.500 +I0916 01:55:53.862574 139728305223424 logging_writer.py:48] [48] global_step=48, grad_norm=0.5, loss=11.289 +I0916 01:55:53.865504 139747222361280 submission.py:307] 48) loss = 11.289, grad_norm = 0.500 +I0916 01:55:56.386042 139728296830720 logging_writer.py:48] [49] global_step=49, grad_norm=0.5, loss=10.9182 +I0916 01:55:56.388913 139747222361280 submission.py:307] 49) loss = 10.918, grad_norm = 0.500 +I0916 01:55:57.964371 139728305223424 logging_writer.py:48] [50] global_step=50, grad_norm=0.5, loss=10.2705 +I0916 01:55:57.967211 139747222361280 submission.py:307] 50) loss = 10.271, grad_norm = 0.500 +I0916 01:55:58.510047 139728296830720 logging_writer.py:48] [51] global_step=51, grad_norm=0.5, loss=9.94725 +I0916 01:55:58.512898 139747222361280 submission.py:307] 51) loss = 9.947, grad_norm = 0.500 +I0916 01:55:59.541134 139728305223424 logging_writer.py:48] [52] global_step=52, grad_norm=0.5, loss=9.42687 +I0916 01:55:59.544058 139747222361280 submission.py:307] 52) loss = 9.427, grad_norm = 0.500 +I0916 01:56:01.827167 139728296830720 logging_writer.py:48] [53] global_step=53, grad_norm=0.5, loss=9.32866 +I0916 01:56:01.830065 139747222361280 submission.py:307] 53) loss = 9.329, grad_norm = 0.500 +I0916 01:56:03.625448 139728305223424 logging_writer.py:48] [54] global_step=54, grad_norm=0.5, loss=9.13858 +I0916 01:56:03.628286 139747222361280 submission.py:307] 54) loss = 9.139, grad_norm = 0.500 +I0916 01:56:04.171632 139728296830720 logging_writer.py:48] [55] global_step=55, grad_norm=0.5, loss=8.82834 +I0916 01:56:04.174470 139747222361280 submission.py:307] 55) loss = 8.828, grad_norm = 0.500 +I0916 01:56:04.814228 139728305223424 logging_writer.py:48] [56] global_step=56, grad_norm=0.5, loss=8.55069 +I0916 01:56:04.817084 139747222361280 submission.py:307] 56) loss = 8.551, grad_norm = 0.500 +I0916 01:56:07.603179 139728296830720 logging_writer.py:48] [57] global_step=57, grad_norm=0.5, loss=8.49077 +I0916 01:56:07.606339 139747222361280 submission.py:307] 57) loss = 8.491, grad_norm = 0.500 +I0916 01:56:08.892994 139728305223424 logging_writer.py:48] [58] global_step=58, grad_norm=0.5, loss=8.23692 +I0916 01:56:08.895774 139747222361280 submission.py:307] 58) loss = 8.237, grad_norm = 0.500 +I0916 01:56:09.436053 139728296830720 logging_writer.py:48] [59] global_step=59, grad_norm=0.5, loss=8.03033 +I0916 01:56:09.438893 139747222361280 submission.py:307] 59) loss = 8.030, grad_norm = 0.500 +I0916 01:56:10.076267 139728305223424 logging_writer.py:48] [60] global_step=60, grad_norm=0.5, loss=7.99374 +I0916 01:56:10.079168 139747222361280 submission.py:307] 60) loss = 7.994, grad_norm = 0.500 +I0916 01:56:13.571287 139728296830720 logging_writer.py:48] [61] global_step=61, grad_norm=0.5, loss=7.86429 +I0916 01:56:13.574255 139747222361280 submission.py:307] 61) loss = 7.864, grad_norm = 0.500 +I0916 01:56:15.117930 139728305223424 logging_writer.py:48] [62] global_step=62, grad_norm=0.5, loss=7.63058 +I0916 01:56:15.120875 139747222361280 submission.py:307] 62) loss = 7.631, grad_norm = 0.500 +I0916 01:56:15.659559 139728296830720 logging_writer.py:48] [63] global_step=63, grad_norm=0.5, loss=7.5257 +I0916 01:56:15.662413 139747222361280 submission.py:307] 63) loss = 7.526, grad_norm = 0.500 +I0916 01:56:16.200713 139728305223424 logging_writer.py:48] [64] global_step=64, grad_norm=0.5, loss=7.28377 +I0916 01:56:16.203636 139747222361280 submission.py:307] 64) loss = 7.284, grad_norm = 0.500 +I0916 01:56:18.796024 139728296830720 logging_writer.py:48] [65] global_step=65, grad_norm=0.5, loss=7.29797 +I0916 01:56:18.798933 139747222361280 submission.py:307] 65) loss = 7.298, grad_norm = 0.500 +I0916 01:56:20.408655 139728305223424 logging_writer.py:48] [66] global_step=66, grad_norm=0.5, loss=7.258 +I0916 01:56:20.411571 139747222361280 submission.py:307] 66) loss = 7.258, grad_norm = 0.500 +I0916 01:56:20.948212 139728296830720 logging_writer.py:48] [67] global_step=67, grad_norm=0.5, loss=7.07914 +I0916 01:56:20.951128 139747222361280 submission.py:307] 67) loss = 7.079, grad_norm = 0.500 +I0916 01:56:21.495548 139728305223424 logging_writer.py:48] [68] global_step=68, grad_norm=0.5, loss=6.98946 +I0916 01:56:21.498838 139747222361280 submission.py:307] 68) loss = 6.989, grad_norm = 0.500 +I0916 01:56:24.211111 139728296830720 logging_writer.py:48] [69] global_step=69, grad_norm=0.5, loss=7.09332 +I0916 01:56:24.214092 139747222361280 submission.py:307] 69) loss = 7.093, grad_norm = 0.500 +I0916 01:56:26.198573 139728305223424 logging_writer.py:48] [70] global_step=70, grad_norm=0.5, loss=6.88386 +I0916 01:56:26.201434 139747222361280 submission.py:307] 70) loss = 6.884, grad_norm = 0.500 +I0916 01:56:26.739516 139728296830720 logging_writer.py:48] [71] global_step=71, grad_norm=0.5, loss=6.85712 +I0916 01:56:26.742534 139747222361280 submission.py:307] 71) loss = 6.857, grad_norm = 0.500 +I0916 01:56:27.285111 139728305223424 logging_writer.py:48] [72] global_step=72, grad_norm=0.5, loss=6.75971 +I0916 01:56:27.288139 139747222361280 submission.py:307] 72) loss = 6.760, grad_norm = 0.500 +I0916 01:56:29.458695 139728296830720 logging_writer.py:48] [73] global_step=73, grad_norm=0.5, loss=6.78967 +I0916 01:56:29.461587 139747222361280 submission.py:307] 73) loss = 6.790, grad_norm = 0.500 +I0916 01:56:31.518369 139728305223424 logging_writer.py:48] [74] global_step=74, grad_norm=0.5, loss=6.71971 +I0916 01:56:31.521227 139747222361280 submission.py:307] 74) loss = 6.720, grad_norm = 0.500 +I0916 01:56:32.069059 139728296830720 logging_writer.py:48] [75] global_step=75, grad_norm=0.5, loss=6.75324 +I0916 01:56:32.072015 139747222361280 submission.py:307] 75) loss = 6.753, grad_norm = 0.500 +I0916 01:56:32.620131 139728305223424 logging_writer.py:48] [76] global_step=76, grad_norm=0.5, loss=6.58454 +I0916 01:56:32.622988 139747222361280 submission.py:307] 76) loss = 6.585, grad_norm = 0.500 +I0916 01:56:34.747853 139728296830720 logging_writer.py:48] [77] global_step=77, grad_norm=0.5, loss=6.6146 +I0916 01:56:34.750812 139747222361280 submission.py:307] 77) loss = 6.615, grad_norm = 0.500 +I0916 01:56:37.073220 139728305223424 logging_writer.py:48] [78] global_step=78, grad_norm=0.5, loss=6.5789 +I0916 01:56:37.076246 139747222361280 submission.py:307] 78) loss = 6.579, grad_norm = 0.500 +I0916 01:56:37.618371 139728296830720 logging_writer.py:48] [79] global_step=79, grad_norm=0.5, loss=6.6254 +I0916 01:56:37.621294 139747222361280 submission.py:307] 79) loss = 6.625, grad_norm = 0.500 +I0916 01:56:38.437585 139728305223424 logging_writer.py:48] [80] global_step=80, grad_norm=0.5, loss=6.58268 +I0916 01:56:38.440451 139747222361280 submission.py:307] 80) loss = 6.583, grad_norm = 0.500 +I0916 01:56:40.213353 139728296830720 logging_writer.py:48] [81] global_step=81, grad_norm=0.5, loss=6.4976 +I0916 01:56:40.216201 139747222361280 submission.py:307] 81) loss = 6.498, grad_norm = 0.500 +I0916 01:56:42.378320 139728305223424 logging_writer.py:48] [82] global_step=82, grad_norm=0.5, loss=6.49011 +I0916 01:56:42.381137 139747222361280 submission.py:307] 82) loss = 6.490, grad_norm = 0.500 +I0916 01:56:42.917947 139728296830720 logging_writer.py:48] [83] global_step=83, grad_norm=0.5, loss=6.45713 +I0916 01:56:42.920969 139747222361280 submission.py:307] 83) loss = 6.457, grad_norm = 0.500 +I0916 01:56:43.565214 139728305223424 logging_writer.py:48] [84] global_step=84, grad_norm=0.5, loss=6.4828 +I0916 01:56:43.568156 139747222361280 submission.py:307] 84) loss = 6.483, grad_norm = 0.500 +I0916 01:56:45.369918 139728296830720 logging_writer.py:48] [85] global_step=85, grad_norm=0.5, loss=6.43188 +I0916 01:56:45.372837 139747222361280 submission.py:307] 85) loss = 6.432, grad_norm = 0.500 +I0916 01:56:47.728029 139728305223424 logging_writer.py:48] [86] global_step=86, grad_norm=0.5, loss=6.38273 +I0916 01:56:47.730895 139747222361280 submission.py:307] 86) loss = 6.383, grad_norm = 0.500 +I0916 01:56:48.273860 139728296830720 logging_writer.py:48] [87] global_step=87, grad_norm=0.5, loss=6.36443 +I0916 01:56:48.276684 139747222361280 submission.py:307] 87) loss = 6.364, grad_norm = 0.500 +I0916 01:56:48.821384 139728305223424 logging_writer.py:48] [88] global_step=88, grad_norm=0.5, loss=6.35718 +I0916 01:56:48.824244 139747222361280 submission.py:307] 88) loss = 6.357, grad_norm = 0.500 +I0916 01:56:51.148704 139728296830720 logging_writer.py:48] [89] global_step=89, grad_norm=0.5, loss=6.32476 +I0916 01:56:51.151607 139747222361280 submission.py:307] 89) loss = 6.325, grad_norm = 0.500 +I0916 01:56:53.375267 139728305223424 logging_writer.py:48] [90] global_step=90, grad_norm=0.5, loss=6.33385 +I0916 01:56:53.378195 139747222361280 submission.py:307] 90) loss = 6.334, grad_norm = 0.500 +I0916 01:56:53.921254 139728296830720 logging_writer.py:48] [91] global_step=91, grad_norm=0.5, loss=6.29919 +I0916 01:56:53.924261 139747222361280 submission.py:307] 91) loss = 6.299, grad_norm = 0.500 +I0916 01:56:54.474349 139728305223424 logging_writer.py:48] [92] global_step=92, grad_norm=0.5, loss=6.27118 +I0916 01:56:54.477192 139747222361280 submission.py:307] 92) loss = 6.271, grad_norm = 0.500 +I0916 01:56:56.634227 139728296830720 logging_writer.py:48] [93] global_step=93, grad_norm=0.5, loss=6.24966 +I0916 01:56:56.637672 139747222361280 submission.py:307] 93) loss = 6.250, grad_norm = 0.500 +I0916 01:56:58.765123 139728305223424 logging_writer.py:48] [94] global_step=94, grad_norm=0.5, loss=6.21637 +I0916 01:56:58.768035 139747222361280 submission.py:307] 94) loss = 6.216, grad_norm = 0.500 +I0916 01:56:59.313556 139728296830720 logging_writer.py:48] [95] global_step=95, grad_norm=0.5, loss=6.23415 +I0916 01:56:59.316478 139747222361280 submission.py:307] 95) loss = 6.234, grad_norm = 0.500 +I0916 01:56:59.862382 139728305223424 logging_writer.py:48] [96] global_step=96, grad_norm=0.5, loss=6.20402 +I0916 01:56:59.865324 139747222361280 submission.py:307] 96) loss = 6.204, grad_norm = 0.500 +I0916 01:57:02.377735 139728296830720 logging_writer.py:48] [97] global_step=97, grad_norm=0.5, loss=6.18195 +I0916 01:57:02.380600 139747222361280 submission.py:307] 97) loss = 6.182, grad_norm = 0.500 +I0916 01:57:04.399101 139728305223424 logging_writer.py:48] [98] global_step=98, grad_norm=0.5, loss=6.16083 +I0916 01:57:04.402105 139747222361280 submission.py:307] 98) loss = 6.161, grad_norm = 0.500 +I0916 01:57:04.943382 139728296830720 logging_writer.py:48] [99] global_step=99, grad_norm=0.499999, loss=6.15441 +I0916 01:57:04.946330 139747222361280 submission.py:307] 99) loss = 6.154, grad_norm = 0.500 +I0916 01:57:05.484143 139728305223424 logging_writer.py:48] [100] global_step=100, grad_norm=0.5, loss=6.14669 +I0916 01:57:05.487092 139747222361280 submission.py:307] 100) loss = 6.147, grad_norm = 0.500 +I0916 02:06:10.173970 139728296830720 logging_writer.py:48] [500] global_step=500, grad_norm=0.5, loss=4.33263 +I0916 02:06:10.177178 139747222361280 submission.py:307] 500) loss = 4.333, grad_norm = 0.500 +I0916 02:17:41.916250 139728305223424 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.5, loss=3.00448 +I0916 02:17:41.919600 139747222361280 submission.py:307] 1000) loss = 3.004, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 02:19:02.599521 139747222361280 spec.py:333] Evaluating on the training split. +I0916 02:19:14.748000 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 02:19:34.476382 139747222361280 spec.py:363] Evaluating on the test split. +I0916 02:19:44.564314 139747222361280 submission_runner.py:516] Time since start: 1597.72s, Step: 1106, {'train/ctc_loss': 6.4069727393639635, 'train/wer': 0.9413664109251888, 'validation/ctc_loss': 6.318610137884268, 'validation/wer': 0.8966832424081495, 'validation/num_examples': 5348, 'test/ctc_loss': 6.257749290276792, 'test/wer': 0.8995389271423638, 'test/num_examples': 2472, 'score': 1480.9474048614502, 'total_duration': 1597.7229297161102, 'accumulated_submission_time': 1480.9474048614502, 'accumulated_eval_time': 114.42702007293701, 'accumulated_logging_time': 0.036197662353515625} +I0916 02:19:44.593803 139728305223424 logging_writer.py:48] [1106] accumulated_eval_time=114.427, accumulated_logging_time=0.0361977, accumulated_submission_time=1480.95, global_step=1106, preemption_count=0, score=1480.95, test/ctc_loss=6.25775, test/num_examples=2472, test/wer=0.899539, total_duration=1597.72, train/ctc_loss=6.40697, train/wer=0.941366, validation/ctc_loss=6.31861, validation/num_examples=5348, validation/wer=0.896683 +I0916 02:23:37.186610 139728296830720 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.5, loss=2.62093 +I0916 02:23:37.190194 139747222361280 submission.py:307] 1500) loss = 2.621, grad_norm = 0.500 +I0916 02:33:25.589934 139728305223424 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.5, loss=2.34825 +I0916 02:33:25.593464 139747222361280 submission.py:307] 2000) loss = 2.348, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 02:38:43.699261 139728305223424 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.5, loss=2.13855 +I0916 02:38:43.707357 139747222361280 submission.py:307] 2500) loss = 2.139, grad_norm = 0.500 +I0916 02:43:52.527316 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 02:44:06.806449 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 02:44:27.173495 139747222361280 spec.py:363] Evaluating on the test split. +I0916 02:44:37.737863 139747222361280 submission_runner.py:516] Time since start: 3090.90s, Step: 2883, {'train/ctc_loss': 4.940423060307439, 'train/wer': 0.9072633120790203, 'validation/ctc_loss': 4.846255274261603, 'validation/wer': 0.8700429681842321, 'validation/num_examples': 5348, 'test/ctc_loss': 4.628748628943803, 'test/wer': 0.864663944914996, 'test/num_examples': 2472, 'score': 2926.629233121872, 'total_duration': 3090.8964488506317, 'accumulated_submission_time': 2926.629233121872, 'accumulated_eval_time': 159.63744711875916, 'accumulated_logging_time': 0.07536149024963379} +I0916 02:44:37.786477 139728305223424 logging_writer.py:48] [2883] accumulated_eval_time=159.637, accumulated_logging_time=0.0753615, accumulated_submission_time=2926.63, global_step=2883, preemption_count=0, score=2926.63, test/ctc_loss=4.62875, test/num_examples=2472, test/wer=0.864664, total_duration=3090.9, train/ctc_loss=4.94042, train/wer=0.907263, validation/ctc_loss=4.84626, validation/num_examples=5348, validation/wer=0.870043 +I0916 02:46:53.166048 139728296830720 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.5, loss=2.06169 +I0916 02:46:53.169534 139747222361280 submission.py:307] 3000) loss = 2.062, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 02:52:26.759490 139728305223424 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.5, loss=1.97339 +I0916 02:52:26.767140 139747222361280 submission.py:307] 3500) loss = 1.973, grad_norm = 0.500 +I0916 02:59:45.848503 139728296830720 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.5, loss=1.83681 +I0916 02:59:45.852404 139747222361280 submission.py:307] 4000) loss = 1.837, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 03:05:30.539545 139728305223424 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.5, loss=1.82781 +I0916 03:05:30.546262 139747222361280 submission.py:307] 4500) loss = 1.828, grad_norm = 0.500 +I0916 03:08:45.454116 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 03:09:01.690114 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 03:09:22.001626 139747222361280 spec.py:363] Evaluating on the test split. +I0916 03:09:32.289819 139747222361280 submission_runner.py:516] Time since start: 4585.45s, Step: 4783, {'train/ctc_loss': 0.6839214356055066, 'train/wer': 0.21452335705612335, 'validation/ctc_loss': 0.9085786806811332, 'validation/wer': 0.2542075025346401, 'validation/num_examples': 5348, 'test/ctc_loss': 0.5881865362281438, 'test/wer': 0.18481506306745477, 'test/num_examples': 2472, 'score': 4372.030340671539, 'total_duration': 4585.448388338089, 'accumulated_submission_time': 4372.030340671539, 'accumulated_eval_time': 206.47299194335938, 'accumulated_logging_time': 0.13427519798278809} +I0916 03:09:32.318791 139728305223424 logging_writer.py:48] [4783] accumulated_eval_time=206.473, accumulated_logging_time=0.134275, accumulated_submission_time=4372.03, global_step=4783, preemption_count=0, score=4372.03, test/ctc_loss=0.588187, test/num_examples=2472, test/wer=0.184815, total_duration=4585.45, train/ctc_loss=0.683921, train/wer=0.214523, validation/ctc_loss=0.908579, validation/num_examples=5348, validation/wer=0.254208 +I0916 03:13:20.993924 139728296830720 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.5, loss=1.76968 +I0916 03:13:20.997719 139747222361280 submission.py:307] 5000) loss = 1.770, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 03:19:26.151206 139728305223424 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.5, loss=1.84732 +I0916 03:19:26.157840 139747222361280 submission.py:307] 5500) loss = 1.847, grad_norm = 0.500 +I0916 03:25:56.024598 139728296830720 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.5, loss=1.70393 +I0916 03:25:56.028296 139747222361280 submission.py:307] 6000) loss = 1.704, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 03:32:15.170238 139728305223424 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.5, loss=1.66859 +I0916 03:32:15.176995 139747222361280 submission.py:307] 6500) loss = 1.669, grad_norm = 0.500 +I0916 03:33:40.378028 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 03:33:55.373587 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 03:34:15.264776 139747222361280 spec.py:363] Evaluating on the test split. +I0916 03:34:25.876192 139747222361280 submission_runner.py:516] Time since start: 6079.03s, Step: 6654, {'train/ctc_loss': 0.49851582776883907, 'train/wer': 0.16415553583194029, 'validation/ctc_loss': 0.736931927883891, 'validation/wer': 0.213334620769565, 'validation/num_examples': 5348, 'test/ctc_loss': 0.45389672317568874, 'test/wer': 0.14638555440456605, 'test/num_examples': 2472, 'score': 5817.834228515625, 'total_duration': 6079.034798622131, 'accumulated_submission_time': 5817.834228515625, 'accumulated_eval_time': 251.97098755836487, 'accumulated_logging_time': 0.17284870147705078} +I0916 03:34:25.905514 139728305223424 logging_writer.py:48] [6654] accumulated_eval_time=251.971, accumulated_logging_time=0.172849, accumulated_submission_time=5817.83, global_step=6654, preemption_count=0, score=5817.83, test/ctc_loss=0.453897, test/num_examples=2472, test/wer=0.146386, total_duration=6079.03, train/ctc_loss=0.498516, train/wer=0.164156, validation/ctc_loss=0.736932, validation/num_examples=5348, validation/wer=0.213335 +I0916 03:39:36.377768 139728296830720 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.5, loss=1.57162 +I0916 03:39:36.387493 139747222361280 submission.py:307] 7000) loss = 1.572, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 03:46:11.515737 139728305223424 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.5, loss=1.58684 +I0916 03:46:11.522930 139747222361280 submission.py:307] 7500) loss = 1.587, grad_norm = 0.500 +I0916 03:52:08.723259 139728296830720 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.5, loss=1.54141 +I0916 03:52:08.726923 139747222361280 submission.py:307] 8000) loss = 1.541, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 03:58:33.973401 139747222361280 spec.py:333] Evaluating on the training split. +I0916 03:58:48.937003 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 03:59:08.720657 139747222361280 spec.py:363] Evaluating on the test split. +I0916 03:59:19.131127 139747222361280 submission_runner.py:516] Time since start: 7572.29s, Step: 8446, {'train/ctc_loss': 0.4237485521826537, 'train/wer': 0.14015867224978082, 'validation/ctc_loss': 0.672041145126959, 'validation/wer': 0.1918794959687153, 'validation/num_examples': 5348, 'test/ctc_loss': 0.39125553463126655, 'test/wer': 0.12761765482501575, 'test/num_examples': 2472, 'score': 7263.678015470505, 'total_duration': 7572.289734840393, 'accumulated_submission_time': 7263.678015470505, 'accumulated_eval_time': 297.12858510017395, 'accumulated_logging_time': 0.21180510520935059} +I0916 03:59:19.160553 139728305223424 logging_writer.py:48] [8446] accumulated_eval_time=297.129, accumulated_logging_time=0.211805, accumulated_submission_time=7263.68, global_step=8446, preemption_count=0, score=7263.68, test/ctc_loss=0.391256, test/num_examples=2472, test/wer=0.127618, total_duration=7572.29, train/ctc_loss=0.423749, train/wer=0.140159, validation/ctc_loss=0.672041, validation/num_examples=5348, validation/wer=0.191879 +I0916 03:59:50.498808 139728296830720 logging_writer.py:48] [8500] global_step=8500, grad_norm=0.5, loss=1.50058 +I0916 03:59:50.502083 139747222361280 submission.py:307] 8500) loss = 1.501, grad_norm = 0.500 +I0916 04:05:52.768221 139728305223424 logging_writer.py:48] [9000] global_step=9000, grad_norm=0.5, loss=1.57726 +I0916 04:05:52.771939 139747222361280 submission.py:307] 9000) loss = 1.577, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 04:13:01.083056 139728305223424 logging_writer.py:48] [9500] global_step=9500, grad_norm=0.5, loss=1.59674 +I0916 04:13:01.089926 139747222361280 submission.py:307] 9500) loss = 1.597, grad_norm = 0.500 +I0916 04:18:36.632043 139728296830720 logging_writer.py:48] [10000] global_step=10000, grad_norm=0.5, loss=1.53381 +I0916 04:18:36.635812 139747222361280 submission.py:307] 10000) loss = 1.534, grad_norm = 0.500 +I0916 04:23:29.361426 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 04:23:43.888795 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 04:24:03.792199 139747222361280 spec.py:363] Evaluating on the test split. +I0916 04:24:14.255734 139747222361280 submission_runner.py:516] Time since start: 9067.41s, Step: 10260, {'train/ctc_loss': 0.37034147229205006, 'train/wer': 0.12410086632652506, 'validation/ctc_loss': 0.6151254756253768, 'validation/wer': 0.17640129387341283, 'validation/num_examples': 5348, 'test/ctc_loss': 0.350934586505581, 'test/wer': 0.11480104807750899, 'test/num_examples': 2472, 'score': 8711.645572900772, 'total_duration': 9067.41434621811, 'accumulated_submission_time': 8711.645572900772, 'accumulated_eval_time': 342.0227270126343, 'accumulated_logging_time': 0.25107884407043457} +I0916 04:24:14.286397 139728305223424 logging_writer.py:48] [10260] accumulated_eval_time=342.023, accumulated_logging_time=0.251079, accumulated_submission_time=8711.65, global_step=10260, preemption_count=0, score=8711.65, test/ctc_loss=0.350935, test/num_examples=2472, test/wer=0.114801, total_duration=9067.41, train/ctc_loss=0.370341, train/wer=0.124101, validation/ctc_loss=0.615125, validation/num_examples=5348, validation/wer=0.176401 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 04:26:45.991120 139728305223424 logging_writer.py:48] [10500] global_step=10500, grad_norm=0.5, loss=1.52093 +I0916 04:26:45.997451 139747222361280 submission.py:307] 10500) loss = 1.521, grad_norm = 0.500 +I0916 04:32:19.108836 139728296830720 logging_writer.py:48] [11000] global_step=11000, grad_norm=0.5, loss=1.51713 +I0916 04:32:19.112377 139747222361280 submission.py:307] 11000) loss = 1.517, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 04:39:50.397551 139728305223424 logging_writer.py:48] [11500] global_step=11500, grad_norm=0.5, loss=1.42194 +I0916 04:39:50.405754 139747222361280 submission.py:307] 11500) loss = 1.422, grad_norm = 0.500 +I0916 04:45:01.945482 139728296830720 logging_writer.py:48] [12000] global_step=12000, grad_norm=0.5, loss=1.52682 +I0916 04:45:01.949185 139747222361280 submission.py:307] 12000) loss = 1.527, grad_norm = 0.500 +I0916 04:48:24.554560 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 04:48:40.969009 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 04:49:00.988155 139747222361280 spec.py:363] Evaluating on the test split. +I0916 04:49:11.473895 139747222361280 submission_runner.py:516] Time since start: 10564.63s, Step: 12195, {'train/ctc_loss': 0.33530889002370934, 'train/wer': 0.11325408796127395, 'validation/ctc_loss': 0.5805158808644766, 'validation/wer': 0.16749867233138607, 'validation/num_examples': 5348, 'test/ctc_loss': 0.32830868120523904, 'test/wer': 0.10614831515446957, 'test/num_examples': 2472, 'score': 10159.301994562149, 'total_duration': 10564.632512331009, 'accumulated_submission_time': 10159.301994562149, 'accumulated_eval_time': 388.9419791698456, 'accumulated_logging_time': 0.29144835472106934} +I0916 04:49:11.574420 139728305223424 logging_writer.py:48] [12195] accumulated_eval_time=388.942, accumulated_logging_time=0.291448, accumulated_submission_time=10159.3, global_step=12195, preemption_count=0, score=10159.3, test/ctc_loss=0.328309, test/num_examples=2472, test/wer=0.106148, total_duration=10564.6, train/ctc_loss=0.335309, train/wer=0.113254, validation/ctc_loss=0.580516, validation/num_examples=5348, validation/wer=0.167499 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 04:53:37.322738 139728305223424 logging_writer.py:48] [12500] global_step=12500, grad_norm=0.5, loss=1.37265 +I0916 04:53:37.330567 139747222361280 submission.py:307] 12500) loss = 1.373, grad_norm = 0.500 +I0916 04:58:46.752695 139728296830720 logging_writer.py:48] [13000] global_step=13000, grad_norm=0.5, loss=1.47685 +I0916 04:58:46.756501 139747222361280 submission.py:307] 13000) loss = 1.477, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 05:06:31.764661 139728305223424 logging_writer.py:48] [13500] global_step=13500, grad_norm=0.5, loss=1.32174 +I0916 05:06:31.771484 139747222361280 submission.py:307] 13500) loss = 1.322, grad_norm = 0.500 +I0916 05:11:27.336974 139728296830720 logging_writer.py:48] [14000] global_step=14000, grad_norm=0.5, loss=1.39735 +I0916 05:11:27.340589 139747222361280 submission.py:307] 14000) loss = 1.397, grad_norm = 0.500 +I0916 05:13:20.167655 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 05:13:35.495556 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 05:13:55.281507 139747222361280 spec.py:363] Evaluating on the test split. +I0916 05:14:05.781849 139747222361280 submission_runner.py:516] Time since start: 12058.94s, Step: 14124, {'train/ctc_loss': 0.314595242569997, 'train/wer': 0.10746587239788945, 'validation/ctc_loss': 0.5686893106728451, 'validation/wer': 0.1635784290059383, 'validation/num_examples': 5348, 'test/ctc_loss': 0.3159095949738693, 'test/wer': 0.10312188978936893, 'test/num_examples': 2472, 'score': 11605.590642929077, 'total_duration': 12058.94044137001, 'accumulated_submission_time': 11605.590642929077, 'accumulated_eval_time': 434.5560202598572, 'accumulated_logging_time': 0.4015920162200928} +I0916 05:14:05.831688 139728305223424 logging_writer.py:48] [14124] accumulated_eval_time=434.556, accumulated_logging_time=0.401592, accumulated_submission_time=11605.6, global_step=14124, preemption_count=0, score=11605.6, test/ctc_loss=0.31591, test/num_examples=2472, test/wer=0.103122, total_duration=12058.9, train/ctc_loss=0.314595, train/wer=0.107466, validation/ctc_loss=0.568689, validation/num_examples=5348, validation/wer=0.163578 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 05:20:22.631560 139728305223424 logging_writer.py:48] [14500] global_step=14500, grad_norm=0.5, loss=1.3542 +I0916 05:20:22.638480 139747222361280 submission.py:307] 14500) loss = 1.354, grad_norm = 0.500 +I0916 05:25:12.371292 139728296830720 logging_writer.py:48] [15000] global_step=15000, grad_norm=0.5, loss=1.35574 +I0916 05:25:12.375046 139747222361280 submission.py:307] 15000) loss = 1.356, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 05:33:14.751139 139728305223424 logging_writer.py:48] [15500] global_step=15500, grad_norm=0.5, loss=1.34106 +I0916 05:33:14.758987 139747222361280 submission.py:307] 15500) loss = 1.341, grad_norm = 0.500 +I0916 05:37:55.633718 139728296830720 logging_writer.py:48] [16000] global_step=16000, grad_norm=0.5, loss=1.34183 +I0916 05:37:55.637394 139747222361280 submission.py:307] 16000) loss = 1.342, grad_norm = 0.500 +I0916 05:38:14.567710 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 05:38:31.115735 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 05:38:51.178658 139747222361280 spec.py:363] Evaluating on the test split. +I0916 05:39:01.954314 139747222361280 submission_runner.py:516] Time since start: 13555.11s, Step: 16026, {'train/ctc_loss': 0.29643379839239126, 'train/wer': 0.10242909027547115, 'validation/ctc_loss': 0.5432022950101718, 'validation/wer': 0.15642350214840922, 'validation/num_examples': 5348, 'test/ctc_loss': 0.3044302253371185, 'test/wer': 0.09883614648711231, 'test/num_examples': 2472, 'score': 13051.919270992279, 'total_duration': 13555.112949848175, 'accumulated_submission_time': 13051.919270992279, 'accumulated_eval_time': 481.94246006011963, 'accumulated_logging_time': 0.46103620529174805} +I0916 05:39:02.024942 139728305223424 logging_writer.py:48] [16026] accumulated_eval_time=481.942, accumulated_logging_time=0.461036, accumulated_submission_time=13051.9, global_step=16026, preemption_count=0, score=13051.9, test/ctc_loss=0.30443, test/num_examples=2472, test/wer=0.0988361, total_duration=13555.1, train/ctc_loss=0.296434, train/wer=0.102429, validation/ctc_loss=0.543202, validation/num_examples=5348, validation/wer=0.156424 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 05:47:08.433421 139728305223424 logging_writer.py:48] [16500] global_step=16500, grad_norm=0.5, loss=1.38647 +I0916 05:47:08.440299 139747222361280 submission.py:307] 16500) loss = 1.386, grad_norm = 0.500 +I0916 05:51:44.803659 139728296830720 logging_writer.py:48] [17000] global_step=17000, grad_norm=0.5, loss=1.26277 +I0916 05:51:44.807721 139747222361280 submission.py:307] 17000) loss = 1.263, grad_norm = 0.500 +I0916 05:59:51.566376 139728305223424 logging_writer.py:48] [17500] global_step=17500, grad_norm=0.5, loss=1.36007 +I0916 05:59:51.570201 139747222361280 submission.py:307] 17500) loss = 1.360, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 06:03:10.107907 139747222361280 spec.py:333] Evaluating on the training split. +I0916 06:03:25.213383 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 06:03:45.052921 139747222361280 spec.py:363] Evaluating on the test split. +I0916 06:03:55.345851 139747222361280 submission_runner.py:516] Time since start: 15048.50s, Step: 17854, {'train/ctc_loss': 0.2800245639965031, 'train/wer': 0.09677700395863849, 'validation/ctc_loss': 0.536387646847373, 'validation/wer': 0.1541350842466084, 'validation/num_examples': 5348, 'test/ctc_loss': 0.29369407401283953, 'test/wer': 0.09426604107001402, 'test/num_examples': 2472, 'score': 14497.74052286148, 'total_duration': 15048.5044631958, 'accumulated_submission_time': 14497.74052286148, 'accumulated_eval_time': 527.1802635192871, 'accumulated_logging_time': 0.5412375926971436} +I0916 06:03:55.393096 139728305223424 logging_writer.py:48] [17854] accumulated_eval_time=527.18, accumulated_logging_time=0.541238, accumulated_submission_time=14497.7, global_step=17854, preemption_count=0, score=14497.7, test/ctc_loss=0.293694, test/num_examples=2472, test/wer=0.094266, total_duration=15048.5, train/ctc_loss=0.280025, train/wer=0.096777, validation/ctc_loss=0.536388, validation/num_examples=5348, validation/wer=0.154135 +I0916 06:05:24.538627 139728296830720 logging_writer.py:48] [18000] global_step=18000, grad_norm=0.5, loss=1.40304 +I0916 06:05:24.542045 139747222361280 submission.py:307] 18000) loss = 1.403, grad_norm = 0.500 +I0916 06:13:33.248611 139728305223424 logging_writer.py:48] [18500] global_step=18500, grad_norm=0.5, loss=1.37788 +I0916 06:13:33.252296 139747222361280 submission.py:307] 18500) loss = 1.378, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 06:18:31.422312 139728305223424 logging_writer.py:48] [19000] global_step=19000, grad_norm=0.5, loss=1.3574 +I0916 06:18:31.429135 139747222361280 submission.py:307] 19000) loss = 1.357, grad_norm = 0.500 +I0916 06:26:14.341577 139728296830720 logging_writer.py:48] [19500] global_step=19500, grad_norm=0.5, loss=1.30002 +I0916 06:26:14.345396 139747222361280 submission.py:307] 19500) loss = 1.300, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 06:28:03.098495 139747222361280 spec.py:333] Evaluating on the training split. +I0916 06:28:18.068040 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 06:28:38.039120 139747222361280 spec.py:363] Evaluating on the test split. +I0916 06:28:48.483178 139747222361280 submission_runner.py:516] Time since start: 16541.64s, Step: 19618, {'train/ctc_loss': 0.2723956721104032, 'train/wer': 0.09438657438918807, 'validation/ctc_loss': 0.5301783712891802, 'validation/wer': 0.15199150292087096, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2908651364604168, 'test/wer': 0.09404261369406698, 'test/num_examples': 2472, 'score': 15943.188425302505, 'total_duration': 16541.64181613922, 'accumulated_submission_time': 15943.188425302505, 'accumulated_eval_time': 572.5648229122162, 'accumulated_logging_time': 0.5980556011199951} +I0916 06:28:48.537400 139728305223424 logging_writer.py:48] [19618] accumulated_eval_time=572.565, accumulated_logging_time=0.598056, accumulated_submission_time=15943.2, global_step=19618, preemption_count=0, score=15943.2, test/ctc_loss=0.290865, test/num_examples=2472, test/wer=0.0940426, total_duration=16541.6, train/ctc_loss=0.272396, train/wer=0.0943866, validation/ctc_loss=0.530178, validation/num_examples=5348, validation/wer=0.151992 +I0916 06:32:19.988386 139728296830720 logging_writer.py:48] [20000] global_step=20000, grad_norm=0.5, loss=1.33857 +I0916 06:32:19.991970 139747222361280 submission.py:307] 20000) loss = 1.339, grad_norm = 0.500 +I0916 06:39:56.125921 139728305223424 logging_writer.py:48] [20500] global_step=20500, grad_norm=0.5, loss=1.39489 +I0916 06:39:56.129668 139747222361280 submission.py:307] 20500) loss = 1.395, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 06:45:26.638658 139728305223424 logging_writer.py:48] [21000] global_step=21000, grad_norm=0.5, loss=1.37939 +I0916 06:45:26.645466 139747222361280 submission.py:307] 21000) loss = 1.379, grad_norm = 0.500 +I0916 06:52:36.999090 139728296830720 logging_writer.py:48] [21500] global_step=21500, grad_norm=0.5, loss=1.35554 +I0916 06:52:37.002794 139747222361280 submission.py:307] 21500) loss = 1.356, grad_norm = 0.500 +I0916 06:52:57.571751 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 06:53:13.742531 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 06:53:33.518107 139747222361280 spec.py:363] Evaluating on the test split. +I0916 06:53:43.867662 139747222361280 submission_runner.py:516] Time since start: 18037.03s, Step: 21517, {'train/ctc_loss': 0.26134596626606843, 'train/wer': 0.09163676360884078, 'validation/ctc_loss': 0.5153653583829365, 'validation/wer': 0.14703809201950466, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2821661881411704, 'test/wer': 0.09071151463449312, 'test/num_examples': 2472, 'score': 17389.614331007004, 'total_duration': 18037.026264190674, 'accumulated_submission_time': 17389.614331007004, 'accumulated_eval_time': 618.8605799674988, 'accumulated_logging_time': 0.6618938446044922} +I0916 06:53:43.958893 139728305223424 logging_writer.py:48] [21517] accumulated_eval_time=618.861, accumulated_logging_time=0.661894, accumulated_submission_time=17389.6, global_step=21517, preemption_count=0, score=17389.6, test/ctc_loss=0.282166, test/num_examples=2472, test/wer=0.0907115, total_duration=18037, train/ctc_loss=0.261346, train/wer=0.0916368, validation/ctc_loss=0.515365, validation/num_examples=5348, validation/wer=0.147038 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 06:59:13.906531 139728305223424 logging_writer.py:48] [22000] global_step=22000, grad_norm=0.5, loss=1.36003 +I0916 06:59:13.913423 139747222361280 submission.py:307] 22000) loss = 1.360, grad_norm = 0.500 +I0916 07:06:04.241425 139728296830720 logging_writer.py:48] [22500] global_step=22500, grad_norm=0.5, loss=1.37342 +I0916 07:06:04.245222 139747222361280 submission.py:307] 22500) loss = 1.373, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 07:12:09.848217 139728305223424 logging_writer.py:48] [23000] global_step=23000, grad_norm=0.5, loss=1.34482 +I0916 07:12:09.855168 139747222361280 submission.py:307] 23000) loss = 1.345, grad_norm = 0.500 +I0916 07:17:52.778117 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 07:18:08.397763 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 07:18:28.159935 139747222361280 spec.py:363] Evaluating on the test split. +I0916 07:18:39.072903 139747222361280 submission_runner.py:516] Time since start: 19532.23s, Step: 23454, {'train/ctc_loss': 0.25362188198396474, 'train/wer': 0.08893595935725215, 'validation/ctc_loss': 0.515249051888688, 'validation/wer': 0.14715396128035532, 'validation/num_examples': 5348, 'test/ctc_loss': 0.27790842856797215, 'test/wer': 0.0894725082769687, 'test/num_examples': 2472, 'score': 18836.100885391235, 'total_duration': 19532.2314286232, 'accumulated_submission_time': 18836.100885391235, 'accumulated_eval_time': 665.1551597118378, 'accumulated_logging_time': 0.762861967086792} +I0916 07:18:39.133888 139728305223424 logging_writer.py:48] [23454] accumulated_eval_time=665.155, accumulated_logging_time=0.762862, accumulated_submission_time=18836.1, global_step=23454, preemption_count=0, score=18836.1, test/ctc_loss=0.277908, test/num_examples=2472, test/wer=0.0894725, total_duration=19532.2, train/ctc_loss=0.253622, train/wer=0.088936, validation/ctc_loss=0.515249, validation/num_examples=5348, validation/wer=0.147154 +I0916 07:19:27.852725 139728296830720 logging_writer.py:48] [23500] global_step=23500, grad_norm=0.5, loss=1.32345 +I0916 07:19:27.856069 139747222361280 submission.py:307] 23500) loss = 1.323, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 07:25:54.603878 139728305223424 logging_writer.py:48] [24000] global_step=24000, grad_norm=0.5, loss=1.32126 +I0916 07:25:54.610666 139747222361280 submission.py:307] 24000) loss = 1.321, grad_norm = 0.500 +I0916 07:32:03.985498 139728296830720 logging_writer.py:48] [24500] global_step=24500, grad_norm=0.5, loss=1.30621 +I0916 07:32:03.989377 139747222361280 submission.py:307] 24500) loss = 1.306, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 07:38:44.047802 139728305223424 logging_writer.py:48] [25000] global_step=25000, grad_norm=0.5, loss=1.3488 +I0916 07:38:44.054497 139747222361280 submission.py:307] 25000) loss = 1.349, grad_norm = 0.500 +I0916 07:42:47.327499 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 07:43:02.992179 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 07:43:22.668743 139747222361280 spec.py:363] Evaluating on the test split. +I0916 07:43:33.162050 139747222361280 submission_runner.py:516] Time since start: 21026.32s, Step: 25378, {'train/ctc_loss': 0.2348187262523939, 'train/wer': 0.08270123986517759, 'validation/ctc_loss': 0.48758527507785815, 'validation/wer': 0.14037560952059094, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2664396442915672, 'test/wer': 0.08662888712855199, 'test/num_examples': 2472, 'score': 20281.97596359253, 'total_duration': 21026.32065486908, 'accumulated_submission_time': 20281.97596359253, 'accumulated_eval_time': 710.9896395206451, 'accumulated_logging_time': 0.8335576057434082} +I0916 07:43:33.222768 139728305223424 logging_writer.py:48] [25378] accumulated_eval_time=710.99, accumulated_logging_time=0.833558, accumulated_submission_time=20282, global_step=25378, preemption_count=0, score=20282, test/ctc_loss=0.26644, test/num_examples=2472, test/wer=0.0866289, total_duration=21026.3, train/ctc_loss=0.234819, train/wer=0.0827012, validation/ctc_loss=0.487585, validation/num_examples=5348, validation/wer=0.140376 +I0916 07:45:35.422027 139728296830720 logging_writer.py:48] [25500] global_step=25500, grad_norm=0.5, loss=1.33112 +I0916 07:45:35.425372 139747222361280 submission.py:307] 25500) loss = 1.331, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 07:52:33.510206 139728305223424 logging_writer.py:48] [26000] global_step=26000, grad_norm=0.5, loss=1.3619 +I0916 07:52:33.516874 139747222361280 submission.py:307] 26000) loss = 1.362, grad_norm = 0.500 +I0916 07:58:13.460187 139728296830720 logging_writer.py:48] [26500] global_step=26500, grad_norm=0.5, loss=1.30388 +I0916 07:58:13.464157 139747222361280 submission.py:307] 26500) loss = 1.304, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 08:05:23.265545 139728305223424 logging_writer.py:48] [27000] global_step=27000, grad_norm=0.5, loss=1.33809 +I0916 08:05:23.281907 139747222361280 submission.py:307] 27000) loss = 1.338, grad_norm = 0.500 +I0916 08:07:41.052700 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 08:07:56.920876 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 08:08:16.619498 139747222361280 spec.py:363] Evaluating on the test split. +I0916 08:08:27.068020 139747222361280 submission_runner.py:516] Time since start: 22520.23s, Step: 27251, {'train/ctc_loss': 0.22198911925932982, 'train/wer': 0.07852479458096695, 'validation/ctc_loss': 0.47852469994788527, 'validation/wer': 0.13673538357553228, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2573770888444416, 'test/wer': 0.08218065118924299, 'test/num_examples': 2472, 'score': 21727.520691156387, 'total_duration': 22520.22661948204, 'accumulated_submission_time': 21727.520691156387, 'accumulated_eval_time': 757.0047874450684, 'accumulated_logging_time': 0.9037110805511475} +I0916 08:08:27.139347 139728305223424 logging_writer.py:48] [27251] accumulated_eval_time=757.005, accumulated_logging_time=0.903711, accumulated_submission_time=21727.5, global_step=27251, preemption_count=0, score=21727.5, test/ctc_loss=0.257377, test/num_examples=2472, test/wer=0.0821807, total_duration=22520.2, train/ctc_loss=0.221989, train/wer=0.0785248, validation/ctc_loss=0.478525, validation/num_examples=5348, validation/wer=0.136735 +I0916 08:11:53.878881 139728296830720 logging_writer.py:48] [27500] global_step=27500, grad_norm=0.5, loss=1.25496 +I0916 08:11:53.883306 139747222361280 submission.py:307] 27500) loss = 1.255, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 08:19:21.563321 139728305223424 logging_writer.py:48] [28000] global_step=28000, grad_norm=0.5, loss=1.2665 +I0916 08:19:21.571991 139747222361280 submission.py:307] 28000) loss = 1.266, grad_norm = 0.500 +I0916 08:24:37.766339 139728296830720 logging_writer.py:48] [28500] global_step=28500, grad_norm=0.5, loss=1.16969 +I0916 08:24:37.770041 139747222361280 submission.py:307] 28500) loss = 1.170, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 08:32:13.454039 139728305223424 logging_writer.py:48] [29000] global_step=29000, grad_norm=0.5, loss=1.19446 +I0916 08:32:13.460934 139747222361280 submission.py:307] 29000) loss = 1.194, grad_norm = 0.500 +I0916 08:32:35.276751 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 08:32:50.325342 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 08:33:10.088095 139747222361280 spec.py:363] Evaluating on the test split. +I0916 08:33:20.458909 139747222361280 submission_runner.py:516] Time since start: 24013.62s, Step: 29040, {'train/ctc_loss': 0.2104059692594999, 'train/wer': 0.07366225789413501, 'validation/ctc_loss': 0.46134533306334136, 'validation/wer': 0.1321102689132429, 'validation/num_examples': 5348, 'test/ctc_loss': 0.24832143908477966, 'test/wer': 0.07874799423151138, 'test/num_examples': 2472, 'score': 23173.42985391617, 'total_duration': 24013.617489099503, 'accumulated_submission_time': 23173.42985391617, 'accumulated_eval_time': 802.1866991519928, 'accumulated_logging_time': 0.9844353199005127} +I0916 08:33:20.536142 139728305223424 logging_writer.py:48] [29040] accumulated_eval_time=802.187, accumulated_logging_time=0.984435, accumulated_submission_time=23173.4, global_step=29040, preemption_count=0, score=23173.4, test/ctc_loss=0.248321, test/num_examples=2472, test/wer=0.078748, total_duration=24013.6, train/ctc_loss=0.210406, train/wer=0.0736623, validation/ctc_loss=0.461345, validation/num_examples=5348, validation/wer=0.13211 +I0916 08:38:22.540190 139728296830720 logging_writer.py:48] [29500] global_step=29500, grad_norm=0.5, loss=1.23675 +I0916 08:38:22.543791 139747222361280 submission.py:307] 29500) loss = 1.237, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 08:46:11.802383 139728305223424 logging_writer.py:48] [30000] global_step=30000, grad_norm=0.5, loss=1.20722 +I0916 08:46:11.809307 139747222361280 submission.py:307] 30000) loss = 1.207, grad_norm = 0.500 +I0916 08:51:11.746825 139728296830720 logging_writer.py:48] [30500] global_step=30500, grad_norm=0.5, loss=1.16991 +I0916 08:51:11.750503 139747222361280 submission.py:307] 30500) loss = 1.170, grad_norm = 0.500 +I0916 08:57:29.635574 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 08:57:44.899731 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 08:58:04.746413 139747222361280 spec.py:363] Evaluating on the test split. +I0916 08:58:15.150569 139747222361280 submission_runner.py:516] Time since start: 25508.31s, Step: 30860, {'train/ctc_loss': 0.19807895260513578, 'train/wer': 0.06955115464826925, 'validation/ctc_loss': 0.45487695547958107, 'validation/wer': 0.12915560276155072, 'validation/num_examples': 5348, 'test/ctc_loss': 0.24187876032324665, 'test/wer': 0.07720431417951375, 'test/num_examples': 2472, 'score': 24620.16385602951, 'total_duration': 25508.309183359146, 'accumulated_submission_time': 24620.16385602951, 'accumulated_eval_time': 847.7015099525452, 'accumulated_logging_time': 1.0710821151733398} +I0916 08:58:15.215353 139728305223424 logging_writer.py:48] [30860] accumulated_eval_time=847.702, accumulated_logging_time=1.07108, accumulated_submission_time=24620.2, global_step=30860, preemption_count=0, score=24620.2, test/ctc_loss=0.241879, test/num_examples=2472, test/wer=0.0772043, total_duration=25508.3, train/ctc_loss=0.198079, train/wer=0.0695512, validation/ctc_loss=0.454877, validation/num_examples=5348, validation/wer=0.129156 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 08:59:53.857991 139728305223424 logging_writer.py:48] [31000] global_step=31000, grad_norm=0.5, loss=1.21202 +I0916 08:59:53.864588 139747222361280 submission.py:307] 31000) loss = 1.212, grad_norm = 0.500 +I0916 09:04:51.641894 139728296830720 logging_writer.py:48] [31500] global_step=31500, grad_norm=0.5, loss=1.13334 +I0916 09:04:51.645934 139747222361280 submission.py:307] 31500) loss = 1.133, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 09:12:53.819701 139728305223424 logging_writer.py:48] [32000] global_step=32000, grad_norm=0.5, loss=1.20063 +I0916 09:12:53.826851 139747222361280 submission.py:307] 32000) loss = 1.201, grad_norm = 0.500 +I0916 09:17:37.749315 139728296830720 logging_writer.py:48] [32500] global_step=32500, grad_norm=0.5, loss=1.19438 +I0916 09:17:37.753099 139747222361280 submission.py:307] 32500) loss = 1.194, grad_norm = 0.500 +I0916 09:22:23.361470 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 09:22:38.408326 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 09:22:58.148218 139747222361280 spec.py:363] Evaluating on the test split. +I0916 09:23:08.381866 139747222361280 submission_runner.py:516] Time since start: 27001.54s, Step: 32795, {'train/ctc_loss': 0.18675809991087738, 'train/wer': 0.06598456838861089, 'validation/ctc_loss': 0.4414691859899287, 'validation/wer': 0.12530294983826581, 'validation/num_examples': 5348, 'test/ctc_loss': 0.23542368156816568, 'test/wer': 0.07436069303109703, 'test/num_examples': 2472, 'score': 26066.00140094757, 'total_duration': 27001.540489673615, 'accumulated_submission_time': 26066.00140094757, 'accumulated_eval_time': 892.7218158245087, 'accumulated_logging_time': 1.1452579498291016} +I0916 09:23:08.444802 139728305223424 logging_writer.py:48] [32795] accumulated_eval_time=892.722, accumulated_logging_time=1.14526, accumulated_submission_time=26066, global_step=32795, preemption_count=0, score=26066, test/ctc_loss=0.235424, test/num_examples=2472, test/wer=0.0743607, total_duration=27001.5, train/ctc_loss=0.186758, train/wer=0.0659846, validation/ctc_loss=0.441469, validation/num_examples=5348, validation/wer=0.125303 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 09:26:39.739872 139728305223424 logging_writer.py:48] [33000] global_step=33000, grad_norm=0.5, loss=1.21333 +I0916 09:26:39.746815 139747222361280 submission.py:307] 33000) loss = 1.213, grad_norm = 0.500 +I0916 09:31:24.259418 139728296830720 logging_writer.py:48] [33500] global_step=33500, grad_norm=0.5, loss=1.18638 +I0916 09:31:24.263057 139747222361280 submission.py:307] 33500) loss = 1.186, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 09:39:35.537189 139728305223424 logging_writer.py:48] [34000] global_step=34000, grad_norm=0.5, loss=1.13345 +I0916 09:39:35.544040 139747222361280 submission.py:307] 34000) loss = 1.133, grad_norm = 0.500 +I0916 09:44:11.212494 139728296830720 logging_writer.py:48] [34500] global_step=34500, grad_norm=0.5, loss=1.12809 +I0916 09:44:11.216381 139747222361280 submission.py:307] 34500) loss = 1.128, grad_norm = 0.500 +I0916 09:47:16.602720 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 09:47:33.573676 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 09:47:53.424854 139747222361280 spec.py:363] Evaluating on the test split. +I0916 09:48:03.882163 139747222361280 submission_runner.py:516] Time since start: 28497.04s, Step: 34725, {'train/ctc_loss': 0.18024144692867128, 'train/wer': 0.06346889991233276, 'validation/ctc_loss': 0.4364551055951125, 'validation/wer': 0.12396079756674552, 'validation/num_examples': 5348, 'test/ctc_loss': 0.23215192310794244, 'test/wer': 0.07387321511993988, 'test/num_examples': 2472, 'score': 27511.845928907394, 'total_duration': 28497.040745019913, 'accumulated_submission_time': 27511.845928907394, 'accumulated_eval_time': 940.0010640621185, 'accumulated_logging_time': 1.217942714691162} +I0916 09:48:03.958711 139728305223424 logging_writer.py:48] [34725] accumulated_eval_time=940.001, accumulated_logging_time=1.21794, accumulated_submission_time=27511.8, global_step=34725, preemption_count=0, score=27511.8, test/ctc_loss=0.232152, test/num_examples=2472, test/wer=0.0738732, total_duration=28497, train/ctc_loss=0.180241, train/wer=0.0634689, validation/ctc_loss=0.436455, validation/num_examples=5348, validation/wer=0.123961 +I0916 09:53:13.897914 139728296830720 logging_writer.py:48] [35000] global_step=35000, grad_norm=0.5, loss=1.13115 +I0916 09:53:13.901836 139747222361280 submission.py:307] 35000) loss = 1.131, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 09:58:01.134472 139728305223424 logging_writer.py:48] [35500] global_step=35500, grad_norm=0.5, loss=1.15683 +I0916 09:58:01.141215 139747222361280 submission.py:307] 35500) loss = 1.157, grad_norm = 0.500 +I0916 10:05:45.644367 139728296830720 logging_writer.py:48] [36000] global_step=36000, grad_norm=0.5, loss=1.18251 +I0916 10:05:45.648117 139747222361280 submission.py:307] 36000) loss = 1.183, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 10:10:47.500179 139728305223424 logging_writer.py:48] [36500] global_step=36500, grad_norm=0.5, loss=1.18808 +I0916 10:10:47.507057 139747222361280 submission.py:307] 36500) loss = 1.188, grad_norm = 0.500 +I0916 10:12:13.050129 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 10:12:29.720802 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 10:12:49.595324 139747222361280 spec.py:363] Evaluating on the test split. +I0916 10:12:59.994013 139747222361280 submission_runner.py:516] Time since start: 29993.15s, Step: 36627, {'train/ctc_loss': 0.17607545084309317, 'train/wer': 0.06197692337012453, 'validation/ctc_loss': 0.43294560250590214, 'validation/wer': 0.1222710375126732, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2286605688270211, 'test/wer': 0.07200454979383747, 'test/num_examples': 2472, 'score': 28958.37071299553, 'total_duration': 29993.15261864662, 'accumulated_submission_time': 28958.37071299553, 'accumulated_eval_time': 986.9447524547577, 'accumulated_logging_time': 1.3041667938232422} +I0916 10:13:00.071002 139728305223424 logging_writer.py:48] [36627] accumulated_eval_time=986.945, accumulated_logging_time=1.30417, accumulated_submission_time=28958.4, global_step=36627, preemption_count=0, score=28958.4, test/ctc_loss=0.228661, test/num_examples=2472, test/wer=0.0720045, total_duration=29993.2, train/ctc_loss=0.176075, train/wer=0.0619769, validation/ctc_loss=0.432946, validation/num_examples=5348, validation/wer=0.122271 +I0916 10:19:22.216829 139728296830720 logging_writer.py:48] [37000] global_step=37000, grad_norm=0.5, loss=1.16057 +I0916 10:19:22.220659 139747222361280 submission.py:307] 37000) loss = 1.161, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 10:24:42.576183 139728305223424 logging_writer.py:48] [37500] global_step=37500, grad_norm=0.5, loss=1.17377 +I0916 10:24:42.582907 139747222361280 submission.py:307] 37500) loss = 1.174, grad_norm = 0.500 +I0916 10:31:53.548462 139728296830720 logging_writer.py:48] [38000] global_step=38000, grad_norm=0.5, loss=1.16758 +I0916 10:31:53.552411 139747222361280 submission.py:307] 38000) loss = 1.168, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 10:37:07.837560 139747222361280 spec.py:333] Evaluating on the training split. +I0916 10:37:23.194665 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 10:37:42.855677 139747222361280 spec.py:363] Evaluating on the test split. +I0916 10:37:53.246031 139747222361280 submission_runner.py:516] Time since start: 31486.40s, Step: 38456, {'train/ctc_loss': 0.17537847614354185, 'train/wer': 0.061573980800331066, 'validation/ctc_loss': 0.43282993371069417, 'validation/wer': 0.12212620093660986, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2286049956852055, 'test/wer': 0.07192330347531127, 'test/num_examples': 2472, 'score': 30403.793350219727, 'total_duration': 31486.40464568138, 'accumulated_submission_time': 30403.793350219727, 'accumulated_eval_time': 1032.3531494140625, 'accumulated_logging_time': 1.3909907341003418} +I0916 10:37:53.309913 139728305223424 logging_writer.py:48] [38456] accumulated_eval_time=1032.35, accumulated_logging_time=1.39099, accumulated_submission_time=30403.8, global_step=38456, preemption_count=0, score=30403.8, test/ctc_loss=0.228605, test/num_examples=2472, test/wer=0.0719233, total_duration=31486.4, train/ctc_loss=0.175378, train/wer=0.061574, validation/ctc_loss=0.43283, validation/num_examples=5348, validation/wer=0.122126 +I0916 10:38:19.271146 139728296830720 logging_writer.py:48] [38500] global_step=38500, grad_norm=0.5, loss=1.13809 +I0916 10:38:19.274466 139747222361280 submission.py:307] 38500) loss = 1.138, grad_norm = 0.500 +I0916 10:45:34.405993 139728305223424 logging_writer.py:48] [39000] global_step=39000, grad_norm=0.5, loss=1.13599 +I0916 10:45:34.409756 139747222361280 submission.py:307] 39000) loss = 1.136, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 10:51:30.584730 139728305223424 logging_writer.py:48] [39500] global_step=39500, grad_norm=0.5, loss=1.17525 +I0916 10:51:30.591670 139747222361280 submission.py:307] 39500) loss = 1.175, grad_norm = 0.500 +I0916 10:58:12.401592 139728296830720 logging_writer.py:48] [40000] global_step=40000, grad_norm=0.5, loss=1.12581 +I0916 10:58:12.405477 139747222361280 submission.py:307] 40000) loss = 1.126, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 11:02:01.239329 139747222361280 spec.py:333] Evaluating on the training split. +I0916 11:02:16.058745 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 11:02:36.054099 139747222361280 spec.py:363] Evaluating on the test split. +I0916 11:02:46.553229 139747222361280 submission_runner.py:516] Time since start: 32979.71s, Step: 40228, {'train/ctc_loss': 0.17460568237481164, 'train/wer': 0.06108391551274442, 'validation/ctc_loss': 0.43228254266500904, 'validation/wer': 0.1220972336213972, 'validation/num_examples': 5348, 'test/ctc_loss': 0.22850715671172334, 'test/wer': 0.0721467308512583, 'test/num_examples': 2472, 'score': 31849.34828710556, 'total_duration': 32979.711805820465, 'accumulated_submission_time': 31849.34828710556, 'accumulated_eval_time': 1077.666915655136, 'accumulated_logging_time': 1.4649040699005127} +I0916 11:02:46.624422 139728305223424 logging_writer.py:48] [40228] accumulated_eval_time=1077.67, accumulated_logging_time=1.4649, accumulated_submission_time=31849.3, global_step=40228, preemption_count=0, score=31849.3, test/ctc_loss=0.228507, test/num_examples=2472, test/wer=0.0721467, total_duration=32979.7, train/ctc_loss=0.174606, train/wer=0.0610839, validation/ctc_loss=0.432283, validation/num_examples=5348, validation/wer=0.122097 +I0916 11:05:15.895039 139728296830720 logging_writer.py:48] [40500] global_step=40500, grad_norm=0.5, loss=1.17239 +I0916 11:05:15.898660 139747222361280 submission.py:307] 40500) loss = 1.172, grad_norm = 0.500 +I0916 11:11:53.519451 139728305223424 logging_writer.py:48] [41000] global_step=41000, grad_norm=0.5, loss=1.14194 +I0916 11:11:53.523059 139747222361280 submission.py:307] 41000) loss = 1.142, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 11:18:22.887327 139728305223424 logging_writer.py:48] [41500] global_step=41500, grad_norm=0.5, loss=1.17558 +I0916 11:18:22.894133 139747222361280 submission.py:307] 41500) loss = 1.176, grad_norm = 0.500 +I0916 11:24:35.561875 139728296830720 logging_writer.py:48] [42000] global_step=42000, grad_norm=0.5, loss=1.1629 +I0916 11:24:35.566346 139747222361280 submission.py:307] 42000) loss = 1.163, grad_norm = 0.500 +I0916 11:26:55.087318 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 11:27:10.722555 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 11:27:30.775441 139747222361280 spec.py:363] Evaluating on the test split. +I0916 11:27:41.179485 139747222361280 submission_runner.py:516] Time since start: 34474.34s, Step: 42123, {'train/ctc_loss': 0.17330378949713754, 'train/wer': 0.061220044759296266, 'validation/ctc_loss': 0.43063256997814947, 'validation/wer': 0.12183652778448317, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2278372798245048, 'test/wer': 0.07249202770499462, 'test/num_examples': 2472, 'score': 33295.38259077072, 'total_duration': 34474.33810329437, 'accumulated_submission_time': 33295.38259077072, 'accumulated_eval_time': 1123.7589085102081, 'accumulated_logging_time': 1.5458810329437256} +I0916 11:27:41.255575 139728305223424 logging_writer.py:48] [42123] accumulated_eval_time=1123.76, accumulated_logging_time=1.54588, accumulated_submission_time=33295.4, global_step=42123, preemption_count=0, score=33295.4, test/ctc_loss=0.227837, test/num_examples=2472, test/wer=0.072492, total_duration=34474.3, train/ctc_loss=0.173304, train/wer=0.06122, validation/ctc_loss=0.430633, validation/num_examples=5348, validation/wer=0.121837 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 11:32:10.690260 139728305223424 logging_writer.py:48] [42500] global_step=42500, grad_norm=0.5, loss=1.18155 +I0916 11:32:10.697005 139747222361280 submission.py:307] 42500) loss = 1.182, grad_norm = 0.500 +I0916 11:38:09.460072 139728296830720 logging_writer.py:48] [43000] global_step=43000, grad_norm=0.5, loss=1.13801 +I0916 11:38:09.464001 139747222361280 submission.py:307] 43000) loss = 1.138, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 11:45:08.137848 139728305223424 logging_writer.py:48] [43500] global_step=43500, grad_norm=0.5, loss=1.12632 +I0916 11:45:08.146405 139747222361280 submission.py:307] 43500) loss = 1.126, grad_norm = 0.500 +I0916 11:50:49.539778 139728296830720 logging_writer.py:48] [44000] global_step=44000, grad_norm=0.5, loss=1.14843 +I0916 11:50:49.543650 139747222361280 submission.py:307] 44000) loss = 1.148, grad_norm = 0.500 +I0916 11:51:50.828456 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 11:52:06.681089 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 11:52:26.424684 139747222361280 spec.py:363] Evaluating on the test split. +I0916 11:52:36.829103 139747222361280 submission_runner.py:516] Time since start: 35969.99s, Step: 44057, {'train/ctc_loss': 0.17360828823343322, 'train/wer': 0.06198236853998661, 'validation/ctc_loss': 0.433137451809574, 'validation/wer': 0.12309177811036547, 'validation/num_examples': 5348, 'test/ctc_loss': 0.22993408848957997, 'test/wer': 0.07356854142546666, 'test/num_examples': 2472, 'score': 34742.66511321068, 'total_duration': 35969.98772978783, 'accumulated_submission_time': 34742.66511321068, 'accumulated_eval_time': 1169.7595500946045, 'accumulated_logging_time': 1.6316978931427002} +I0916 11:52:36.906266 139728305223424 logging_writer.py:48] [44057] accumulated_eval_time=1169.76, accumulated_logging_time=1.6317, accumulated_submission_time=34742.7, global_step=44057, preemption_count=0, score=34742.7, test/ctc_loss=0.229934, test/num_examples=2472, test/wer=0.0735685, total_duration=35970, train/ctc_loss=0.173608, train/wer=0.0619824, validation/ctc_loss=0.433137, validation/num_examples=5348, validation/wer=0.123092 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 11:58:54.909794 139728305223424 logging_writer.py:48] [44500] global_step=44500, grad_norm=0.5, loss=1.19366 +I0916 11:58:54.916725 139747222361280 submission.py:307] 44500) loss = 1.194, grad_norm = 0.500 +I0916 12:04:20.318637 139728296830720 logging_writer.py:48] [45000] global_step=45000, grad_norm=0.5, loss=1.20186 +I0916 12:04:20.322353 139747222361280 submission.py:307] 45000) loss = 1.202, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 12:11:47.575151 139728305223424 logging_writer.py:48] [45500] global_step=45500, grad_norm=0.5, loss=1.23033 +I0916 12:11:47.582175 139747222361280 submission.py:307] 45500) loss = 1.230, grad_norm = 0.500 +I0916 12:16:45.365118 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 12:17:01.481280 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 12:17:21.526285 139747222361280 spec.py:363] Evaluating on the test split. +I0916 12:17:31.976453 139747222361280 submission_runner.py:516] Time since start: 37465.14s, Step: 45979, {'train/ctc_loss': 0.17677941482241807, 'train/wer': 0.062281852882400665, 'validation/ctc_loss': 0.4403990288922795, 'validation/wer': 0.12456911118621156, 'validation/num_examples': 5348, 'test/ctc_loss': 0.23365819468191495, 'test/wer': 0.07413726565515, 'test/num_examples': 2472, 'score': 36188.63485765457, 'total_duration': 37465.13506793976, 'accumulated_submission_time': 36188.63485765457, 'accumulated_eval_time': 1216.3707485198975, 'accumulated_logging_time': 1.7183234691619873} +I0916 12:17:32.027780 139728305223424 logging_writer.py:48] [45979] accumulated_eval_time=1216.37, accumulated_logging_time=1.71832, accumulated_submission_time=36188.6, global_step=45979, preemption_count=0, score=36188.6, test/ctc_loss=0.233658, test/num_examples=2472, test/wer=0.0741373, total_duration=37465.1, train/ctc_loss=0.176779, train/wer=0.0622819, validation/ctc_loss=0.440399, validation/num_examples=5348, validation/wer=0.124569 +I0916 12:17:49.926397 139728296830720 logging_writer.py:48] [46000] global_step=46000, grad_norm=0.5, loss=1.11965 +I0916 12:17:49.929760 139747222361280 submission.py:307] 46000) loss = 1.120, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 12:25:38.039506 139728305223424 logging_writer.py:48] [46500] global_step=46500, grad_norm=0.5, loss=1.21185 +I0916 12:25:38.046495 139747222361280 submission.py:307] 46500) loss = 1.212, grad_norm = 0.500 +I0916 12:30:41.700817 139728296830720 logging_writer.py:48] [47000] global_step=47000, grad_norm=0.5, loss=1.17694 +I0916 12:30:41.704596 139747222361280 submission.py:307] 47000) loss = 1.177, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 12:38:25.367258 139728305223424 logging_writer.py:48] [47500] global_step=47500, grad_norm=0.5, loss=1.17632 +I0916 12:38:25.374189 139747222361280 submission.py:307] 47500) loss = 1.176, grad_norm = 0.500 +I0916 12:41:39.900951 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 12:41:55.346627 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 12:42:15.289911 139747222361280 spec.py:363] Evaluating on the test split. +I0916 12:42:25.511710 139747222361280 submission_runner.py:516] Time since start: 38958.67s, Step: 47856, {'train/ctc_loss': 0.18668259974419768, 'train/wer': 0.06670333081040464, 'validation/ctc_loss': 0.46038853923359957, 'validation/wer': 0.13012117993530634, 'validation/num_examples': 5348, 'test/ctc_loss': 0.24082118201174268, 'test/wer': 0.07661527837019885, 'test/num_examples': 2472, 'score': 37634.08888673782, 'total_duration': 38958.67030453682, 'accumulated_submission_time': 37634.08888673782, 'accumulated_eval_time': 1261.9813685417175, 'accumulated_logging_time': 1.7793095111846924} +I0916 12:42:25.574270 139728305223424 logging_writer.py:48] [47856] accumulated_eval_time=1261.98, accumulated_logging_time=1.77931, accumulated_submission_time=37634.1, global_step=47856, preemption_count=0, score=37634.1, test/ctc_loss=0.240821, test/num_examples=2472, test/wer=0.0766153, total_duration=38958.7, train/ctc_loss=0.186683, train/wer=0.0667033, validation/ctc_loss=0.460389, validation/num_examples=5348, validation/wer=0.130121 +I0916 12:44:15.186135 139728296830720 logging_writer.py:48] [48000] global_step=48000, grad_norm=0.5, loss=1.20617 +I0916 12:44:15.189801 139747222361280 submission.py:307] 48000) loss = 1.206, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 12:52:20.500128 139728305223424 logging_writer.py:48] [48500] global_step=48500, grad_norm=0.5, loss=1.25818 +I0916 12:52:20.506998 139747222361280 submission.py:307] 48500) loss = 1.258, grad_norm = 0.500 +I0916 12:57:08.024215 139728296830720 logging_writer.py:48] [49000] global_step=49000, grad_norm=0.5, loss=1.27499 +I0916 12:57:08.028216 139747222361280 submission.py:307] 49000) loss = 1.275, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 13:05:11.820488 139728305223424 logging_writer.py:48] [49500] global_step=49500, grad_norm=0.5, loss=1.18124 +I0916 13:05:11.827137 139747222361280 submission.py:307] 49500) loss = 1.181, grad_norm = 0.500 +I0916 13:06:33.711464 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 13:06:48.986793 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 13:07:08.780198 139747222361280 spec.py:363] Evaluating on the test split. +I0916 13:07:19.091760 139747222361280 submission_runner.py:516] Time since start: 40452.25s, Step: 49651, {'train/ctc_loss': 0.19449783699064893, 'train/wer': 0.07053673039330462, 'validation/ctc_loss': 0.47208595084262606, 'validation/wer': 0.1332979288369623, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2506045752951803, 'test/wer': 0.08112444904840249, 'test/num_examples': 2472, 'score': 39079.884689092636, 'total_duration': 40452.2503619194, 'accumulated_submission_time': 39079.884689092636, 'accumulated_eval_time': 1307.3615272045135, 'accumulated_logging_time': 1.8513941764831543} +I0916 13:07:19.186918 139728305223424 logging_writer.py:48] [49651] accumulated_eval_time=1307.36, accumulated_logging_time=1.85139, accumulated_submission_time=39079.9, global_step=49651, preemption_count=0, score=39079.9, test/ctc_loss=0.250605, test/num_examples=2472, test/wer=0.0811244, total_duration=40452.3, train/ctc_loss=0.194498, train/wer=0.0705367, validation/ctc_loss=0.472086, validation/num_examples=5348, validation/wer=0.133298 +I0916 13:10:52.270994 139728296830720 logging_writer.py:48] [50000] global_step=50000, grad_norm=0.5, loss=1.27703 +I0916 13:10:52.274768 139747222361280 submission.py:307] 50000) loss = 1.277, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 13:19:08.389070 139728305223424 logging_writer.py:48] [50500] global_step=50500, grad_norm=0.5, loss=1.10135 +I0916 13:19:08.395993 139747222361280 submission.py:307] 50500) loss = 1.101, grad_norm = 0.500 +I0916 13:23:47.014605 139728296830720 logging_writer.py:48] [51000] global_step=51000, grad_norm=0.5, loss=1.17124 +I0916 13:23:47.018383 139747222361280 submission.py:307] 51000) loss = 1.171, grad_norm = 0.500 +I0916 13:31:27.273215 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 13:31:42.823611 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 13:32:02.775666 139747222361280 spec.py:363] Evaluating on the test split. +I0916 13:32:13.359467 139747222361280 submission_runner.py:516] Time since start: 41946.52s, Step: 51465, {'train/ctc_loss': 0.2047566513494253, 'train/wer': 0.07402163910503189, 'validation/ctc_loss': 0.4808824695160237, 'validation/wer': 0.13502631197798484, 'validation/num_examples': 5348, 'test/ctc_loss': 0.25553584102200144, 'test/wer': 0.08315560701155729, 'test/num_examples': 2472, 'score': 40525.517881155014, 'total_duration': 41946.51808142662, 'accumulated_submission_time': 40525.517881155014, 'accumulated_eval_time': 1353.447606086731, 'accumulated_logging_time': 1.9562325477600098} +I0916 13:32:13.422158 139728305223424 logging_writer.py:48] [51465] accumulated_eval_time=1353.45, accumulated_logging_time=1.95623, accumulated_submission_time=40525.5, global_step=51465, preemption_count=0, score=40525.5, test/ctc_loss=0.255536, test/num_examples=2472, test/wer=0.0831556, total_duration=41946.5, train/ctc_loss=0.204757, train/wer=0.0740216, validation/ctc_loss=0.480882, validation/num_examples=5348, validation/wer=0.135026 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 13:32:52.103161 139728305223424 logging_writer.py:48] [51500] global_step=51500, grad_norm=0.5, loss=1.22931 +I0916 13:32:52.111591 139747222361280 submission.py:307] 51500) loss = 1.229, grad_norm = 0.500 +I0916 13:37:30.448491 139728296830720 logging_writer.py:48] [52000] global_step=52000, grad_norm=0.5, loss=1.20782 +I0916 13:37:30.452321 139747222361280 submission.py:307] 52000) loss = 1.208, grad_norm = 0.500 +I0916 13:45:31.673116 139728305223424 logging_writer.py:48] [52500] global_step=52500, grad_norm=0.5, loss=1.19398 +I0916 13:45:31.676795 139747222361280 submission.py:307] 52500) loss = 1.194, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 13:50:22.142757 139728305223424 logging_writer.py:48] [53000] global_step=53000, grad_norm=0.5, loss=1.29896 +I0916 13:50:22.149604 139747222361280 submission.py:307] 53000) loss = 1.299, grad_norm = 0.500 +I0916 13:56:21.113576 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 13:56:36.566087 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 13:56:56.302510 139747222361280 spec.py:363] Evaluating on the test split. +I0916 13:57:06.826886 139747222361280 submission_runner.py:516] Time since start: 43439.99s, Step: 53400, {'train/ctc_loss': 0.22536685480496774, 'train/wer': 0.08064841082717575, 'validation/ctc_loss': 0.49898532271888185, 'validation/wer': 0.1406942499879303, 'validation/num_examples': 5348, 'test/ctc_loss': 0.26879735688592815, 'test/wer': 0.0860398513192371, 'test/num_examples': 2472, 'score': 41970.87526202202, 'total_duration': 43439.98548531532, 'accumulated_submission_time': 41970.87526202202, 'accumulated_eval_time': 1399.1608033180237, 'accumulated_logging_time': 2.0289409160614014} +I0916 13:57:06.890704 139728305223424 logging_writer.py:48] [53400] accumulated_eval_time=1399.16, accumulated_logging_time=2.02894, accumulated_submission_time=41970.9, global_step=53400, preemption_count=0, score=41970.9, test/ctc_loss=0.268797, test/num_examples=2472, test/wer=0.0860399, total_duration=43440, train/ctc_loss=0.225367, train/wer=0.0806484, validation/ctc_loss=0.498985, validation/num_examples=5348, validation/wer=0.140694 +I0916 13:58:59.048907 139728296830720 logging_writer.py:48] [53500] global_step=53500, grad_norm=0.5, loss=1.28189 +I0916 13:58:59.052657 139747222361280 submission.py:307] 53500) loss = 1.282, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 14:04:12.114842 139728305223424 logging_writer.py:48] [54000] global_step=54000, grad_norm=0.5, loss=1.27681 +I0916 14:04:12.121627 139747222361280 submission.py:307] 54000) loss = 1.277, grad_norm = 0.500 +I0916 14:11:38.890034 139728296830720 logging_writer.py:48] [54500] global_step=54500, grad_norm=0.5, loss=1.34542 +I0916 14:11:38.893827 139747222361280 submission.py:307] 54500) loss = 1.345, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 14:17:03.773688 139728305223424 logging_writer.py:48] [55000] global_step=55000, grad_norm=0.5, loss=1.32205 +I0916 14:17:03.780321 139747222361280 submission.py:307] 55000) loss = 1.322, grad_norm = 0.500 +I0916 14:21:14.767714 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 14:21:30.587957 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 14:21:50.450595 139747222361280 spec.py:363] Evaluating on the test split. +I0916 14:22:00.984965 139747222361280 submission_runner.py:516] Time since start: 44934.14s, Step: 55331, {'train/ctc_loss': 0.240866233795501, 'train/wer': 0.08560351540166296, 'validation/ctc_loss': 0.5176834925030138, 'validation/wer': 0.14591802249794814, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2769799916526873, 'test/wer': 0.08776633558791867, 'test/num_examples': 2472, 'score': 43416.445709466934, 'total_duration': 44934.14357638359, 'accumulated_submission_time': 43416.445709466934, 'accumulated_eval_time': 1445.3778800964355, 'accumulated_logging_time': 2.102470874786377} +I0916 14:22:01.048467 139728305223424 logging_writer.py:48] [55331] accumulated_eval_time=1445.38, accumulated_logging_time=2.10247, accumulated_submission_time=43416.4, global_step=55331, preemption_count=0, score=43416.4, test/ctc_loss=0.27698, test/num_examples=2472, test/wer=0.0877663, total_duration=44934.1, train/ctc_loss=0.240866, train/wer=0.0856035, validation/ctc_loss=0.517683, validation/num_examples=5348, validation/wer=0.145918 +I0916 14:25:05.827000 139728296830720 logging_writer.py:48] [55500] global_step=55500, grad_norm=0.5, loss=1.28281 +I0916 14:25:05.830643 139747222361280 submission.py:307] 55500) loss = 1.283, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 14:30:54.085170 139728305223424 logging_writer.py:48] [56000] global_step=56000, grad_norm=0.5, loss=1.33741 +I0916 14:30:54.091874 139747222361280 submission.py:307] 56000) loss = 1.337, grad_norm = 0.500 +I0916 14:37:42.755836 139728296830720 logging_writer.py:48] [56500] global_step=56500, grad_norm=0.5, loss=1.3312 +I0916 14:37:42.759680 139747222361280 submission.py:307] 56500) loss = 1.331, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 14:43:44.791947 139728305223424 logging_writer.py:48] [57000] global_step=57000, grad_norm=0.5, loss=1.26408 +I0916 14:43:44.799228 139747222361280 submission.py:307] 57000) loss = 1.264, grad_norm = 0.500 +I0916 14:46:09.687177 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 14:46:26.293795 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 14:46:46.584416 139747222361280 spec.py:363] Evaluating on the test split. +I0916 14:46:56.906221 139747222361280 submission_runner.py:516] Time since start: 46430.06s, Step: 57234, {'train/ctc_loss': 0.25012288376301545, 'train/wer': 0.08944780532428709, 'validation/ctc_loss': 0.5302517557326201, 'validation/wer': 0.1483512769758123, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2789501036357184, 'test/wer': 0.08985842828996811, 'test/num_examples': 2472, 'score': 44862.44470882416, 'total_duration': 46430.064829826355, 'accumulated_submission_time': 44862.44470882416, 'accumulated_eval_time': 1492.5967779159546, 'accumulated_logging_time': 2.17545223236084} +I0916 14:46:56.976220 139728305223424 logging_writer.py:48] [57234] accumulated_eval_time=1492.6, accumulated_logging_time=2.17545, accumulated_submission_time=44862.4, global_step=57234, preemption_count=0, score=44862.4, test/ctc_loss=0.27895, test/num_examples=2472, test/wer=0.0898584, total_duration=46430.1, train/ctc_loss=0.250123, train/wer=0.0894478, validation/ctc_loss=0.530252, validation/num_examples=5348, validation/wer=0.148351 +I0916 14:51:21.363988 139728296830720 logging_writer.py:48] [57500] global_step=57500, grad_norm=0.5, loss=1.31888 +I0916 14:51:21.367769 139747222361280 submission.py:307] 57500) loss = 1.319, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 14:57:38.447228 139728305223424 logging_writer.py:48] [58000] global_step=58000, grad_norm=0.5, loss=1.32746 +I0916 14:57:38.454189 139747222361280 submission.py:307] 58000) loss = 1.327, grad_norm = 0.500 +I0916 15:03:52.827900 139728296830720 logging_writer.py:48] [58500] global_step=58500, grad_norm=0.5, loss=1.3228 +I0916 15:03:52.831862 139747222361280 submission.py:307] 58500) loss = 1.323, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 15:10:26.609603 139728305223424 logging_writer.py:48] [59000] global_step=59000, grad_norm=0.5, loss=1.24786 +I0916 15:10:26.616652 139747222361280 submission.py:307] 59000) loss = 1.248, grad_norm = 0.500 +I0916 15:11:04.731364 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 15:11:19.917937 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 15:11:39.791934 139747222361280 spec.py:363] Evaluating on the test split. +I0916 15:11:50.172481 139747222361280 submission_runner.py:516] Time since start: 47923.33s, Step: 59070, {'train/ctc_loss': 0.2561507256520068, 'train/wer': 0.09059673616518467, 'validation/ctc_loss': 0.5335001455915963, 'validation/wer': 0.1492685752908801, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2836022434915156, 'test/wer': 0.09195052099201755, 'test/num_examples': 2472, 'score': 46307.860738277435, 'total_duration': 47923.331095695496, 'accumulated_submission_time': 46307.860738277435, 'accumulated_eval_time': 1538.037742137909, 'accumulated_logging_time': 2.2553555965423584} +I0916 15:11:50.246123 139728305223424 logging_writer.py:48] [59070] accumulated_eval_time=1538.04, accumulated_logging_time=2.25536, accumulated_submission_time=46307.9, global_step=59070, preemption_count=0, score=46307.9, test/ctc_loss=0.283602, test/num_examples=2472, test/wer=0.0919505, total_duration=47923.3, train/ctc_loss=0.256151, train/wer=0.0905967, validation/ctc_loss=0.5335, validation/num_examples=5348, validation/wer=0.149269 +I0916 15:17:37.328573 139728296830720 logging_writer.py:48] [59500] global_step=59500, grad_norm=0.5, loss=1.29232 +I0916 15:17:37.332276 139747222361280 submission.py:307] 59500) loss = 1.292, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 15:24:27.989346 139728305223424 logging_writer.py:48] [60000] global_step=60000, grad_norm=0.5, loss=1.32777 +I0916 15:24:27.996188 139747222361280 submission.py:307] 60000) loss = 1.328, grad_norm = 0.500 +I0916 15:30:14.748804 139728296830720 logging_writer.py:48] [60500] global_step=60500, grad_norm=0.5, loss=1.36136 +I0916 15:30:14.752539 139747222361280 submission.py:307] 60500) loss = 1.361, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 15:35:57.973187 139747222361280 spec.py:333] Evaluating on the training split. +I0916 15:36:12.983512 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 15:36:32.758852 139747222361280 spec.py:363] Evaluating on the test split. +I0916 15:36:43.306221 139747222361280 submission_runner.py:516] Time since start: 49416.46s, Step: 60843, {'train/ctc_loss': 0.27320185762831783, 'train/wer': 0.09704926245174218, 'validation/ctc_loss': 0.5415587974212628, 'validation/wer': 0.1533239994206537, 'validation/num_examples': 5348, 'test/ctc_loss': 0.29504484160268407, 'test/wer': 0.09566754006459083, 'test/num_examples': 2472, 'score': 47753.28298354149, 'total_duration': 49416.464834451675, 'accumulated_submission_time': 47753.28298354149, 'accumulated_eval_time': 1583.370682001114, 'accumulated_logging_time': 2.3385322093963623} +I0916 15:36:43.364152 139728305223424 logging_writer.py:48] [60843] accumulated_eval_time=1583.37, accumulated_logging_time=2.33853, accumulated_submission_time=47753.3, global_step=60843, preemption_count=0, score=47753.3, test/ctc_loss=0.295045, test/num_examples=2472, test/wer=0.0956675, total_duration=49416.5, train/ctc_loss=0.273202, train/wer=0.0970493, validation/ctc_loss=0.541559, validation/num_examples=5348, validation/wer=0.153324 +I0916 15:38:10.366442 139728296830720 logging_writer.py:48] [61000] global_step=61000, grad_norm=0.5, loss=1.39616 +I0916 15:38:10.369762 139747222361280 submission.py:307] 61000) loss = 1.396, grad_norm = 0.500 +I0916 15:43:57.036454 139728305223424 logging_writer.py:48] [61500] global_step=61500, grad_norm=0.5, loss=1.29709 +I0916 15:43:57.040160 139747222361280 submission.py:307] 61500) loss = 1.297, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 15:51:17.661431 139728305223424 logging_writer.py:48] [62000] global_step=62000, grad_norm=0.5, loss=1.32862 +I0916 15:51:17.668179 139747222361280 submission.py:307] 62000) loss = 1.329, grad_norm = 0.500 +I0916 15:56:39.269644 139728296830720 logging_writer.py:48] [62500] global_step=62500, grad_norm=0.5, loss=1.41654 +I0916 15:56:39.273507 139747222361280 submission.py:307] 62500) loss = 1.417, grad_norm = 0.500 +I0916 16:00:51.376029 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 16:01:07.590506 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 16:01:27.436642 139747222361280 spec.py:363] Evaluating on the test split. +I0916 16:01:38.128938 139747222361280 submission_runner.py:516] Time since start: 50911.29s, Step: 62733, {'train/ctc_loss': 0.2780718176932209, 'train/wer': 0.09734330162429417, 'validation/ctc_loss': 0.5457853195009544, 'validation/wer': 0.15320813015980303, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2982510716417188, 'test/wer': 0.09483476529969735, 'test/num_examples': 2472, 'score': 49198.828130960464, 'total_duration': 50911.28756451607, 'accumulated_submission_time': 49198.828130960464, 'accumulated_eval_time': 1630.123468875885, 'accumulated_logging_time': 2.4058196544647217} +I0916 16:01:38.230771 139728305223424 logging_writer.py:48] [62733] accumulated_eval_time=1630.12, accumulated_logging_time=2.40582, accumulated_submission_time=49198.8, global_step=62733, preemption_count=0, score=49198.8, test/ctc_loss=0.298251, test/num_examples=2472, test/wer=0.0948348, total_duration=50911.3, train/ctc_loss=0.278072, train/wer=0.0973433, validation/ctc_loss=0.545785, validation/num_examples=5348, validation/wer=0.153208 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 16:05:02.968394 139728305223424 logging_writer.py:48] [63000] global_step=63000, grad_norm=0.5, loss=1.34993 +I0916 16:05:02.977020 139747222361280 submission.py:307] 63000) loss = 1.350, grad_norm = 0.500 +I0916 16:10:21.842355 139728296830720 logging_writer.py:48] [63500] global_step=63500, grad_norm=0.5, loss=1.3608 +I0916 16:10:21.846183 139747222361280 submission.py:307] 63500) loss = 1.361, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 16:18:01.138284 139728305223424 logging_writer.py:48] [64000] global_step=64000, grad_norm=0.5, loss=1.41671 +I0916 16:18:01.145352 139747222361280 submission.py:307] 64000) loss = 1.417, grad_norm = 0.500 +I0916 16:23:03.079296 139728296830720 logging_writer.py:48] [64500] global_step=64500, grad_norm=0.5, loss=1.39188 +I0916 16:23:03.083130 139747222361280 submission.py:307] 64500) loss = 1.392, grad_norm = 0.500 +I0916 16:25:46.186867 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 16:26:02.017792 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 16:26:22.074797 139747222361280 spec.py:363] Evaluating on the test split. +I0916 16:26:32.704473 139747222361280 submission_runner.py:516] Time since start: 52405.86s, Step: 64663, {'train/ctc_loss': 0.27760106445402194, 'train/wer': 0.09670621675043153, 'validation/ctc_loss': 0.546718027928471, 'validation/wer': 0.154666151692174, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2912591638976708, 'test/wer': 0.0934942010440152, 'test/num_examples': 2472, 'score': 50644.459433078766, 'total_duration': 52405.86308503151, 'accumulated_submission_time': 50644.459433078766, 'accumulated_eval_time': 1676.6409544944763, 'accumulated_logging_time': 2.5173280239105225} +I0916 16:26:32.760967 139728305223424 logging_writer.py:48] [64663] accumulated_eval_time=1676.64, accumulated_logging_time=2.51733, accumulated_submission_time=50644.5, global_step=64663, preemption_count=0, score=50644.5, test/ctc_loss=0.291259, test/num_examples=2472, test/wer=0.0934942, total_duration=52405.9, train/ctc_loss=0.277601, train/wer=0.0967062, validation/ctc_loss=0.546718, validation/num_examples=5348, validation/wer=0.154666 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 16:31:49.853232 139728305223424 logging_writer.py:48] [65000] global_step=65000, grad_norm=0.5, loss=1.34272 +I0916 16:31:49.859954 139747222361280 submission.py:307] 65000) loss = 1.343, grad_norm = 0.500 +I0916 16:36:50.064268 139728296830720 logging_writer.py:48] [65500] global_step=65500, grad_norm=0.5, loss=1.3381 +I0916 16:36:50.068132 139747222361280 submission.py:307] 65500) loss = 1.338, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 16:44:45.073127 139728305223424 logging_writer.py:48] [66000] global_step=66000, grad_norm=0.5, loss=1.29967 +I0916 16:44:45.079941 139747222361280 submission.py:307] 66000) loss = 1.300, grad_norm = 0.500 +I0916 16:49:32.351626 139728296830720 logging_writer.py:48] [66500] global_step=66500, grad_norm=0.5, loss=1.3684 +I0916 16:49:32.355696 139747222361280 submission.py:307] 66500) loss = 1.368, grad_norm = 0.500 +I0916 16:50:40.549390 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 16:50:56.634449 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 16:51:16.580979 139747222361280 spec.py:363] Evaluating on the test split. +I0916 16:51:27.098116 139747222361280 submission_runner.py:516] Time since start: 53900.26s, Step: 66583, {'train/ctc_loss': 0.2903884168659453, 'train/wer': 0.10067574557988336, 'validation/ctc_loss': 0.5558291774977396, 'validation/wer': 0.15788152368078018, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2992008052938899, 'test/wer': 0.09680498852395751, 'test/num_examples': 2472, 'score': 52089.85448694229, 'total_duration': 53900.25670194626, 'accumulated_submission_time': 52089.85448694229, 'accumulated_eval_time': 1723.1894779205322, 'accumulated_logging_time': 2.5836801528930664} +I0916 16:51:27.151910 139728305223424 logging_writer.py:48] [66583] accumulated_eval_time=1723.19, accumulated_logging_time=2.58368, accumulated_submission_time=52089.9, global_step=66583, preemption_count=0, score=52089.9, test/ctc_loss=0.299201, test/num_examples=2472, test/wer=0.096805, total_duration=53900.3, train/ctc_loss=0.290388, train/wer=0.100676, validation/ctc_loss=0.555829, validation/num_examples=5348, validation/wer=0.157882 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 16:58:34.997726 139728305223424 logging_writer.py:48] [67000] global_step=67000, grad_norm=0.5, loss=1.3701 +I0916 16:58:35.004370 139747222361280 submission.py:307] 67000) loss = 1.370, grad_norm = 0.500 +I0916 17:03:15.372658 139728296830720 logging_writer.py:48] [67500] global_step=67500, grad_norm=0.5, loss=1.41771 +I0916 17:03:15.376707 139747222361280 submission.py:307] 67500) loss = 1.418, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 17:11:21.603839 139728305223424 logging_writer.py:48] [68000] global_step=68000, grad_norm=0.5, loss=1.37684 +I0916 17:11:21.610816 139747222361280 submission.py:307] 68000) loss = 1.377, grad_norm = 0.500 +I0916 17:15:35.366531 139747222361280 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 17:15:50.849531 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 17:16:10.510484 139747222361280 spec.py:363] Evaluating on the test split. +I0916 17:16:20.958361 139747222361280 submission_runner.py:516] Time since start: 55394.12s, Step: 68464, {'train/ctc_loss': 0.2889195519547448, 'train/wer': 0.1000223251964345, 'validation/ctc_loss': 0.5444093611864577, 'validation/wer': 0.15438613431178486, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2996781042325311, 'test/wer': 0.09749558223143014, 'test/num_examples': 2472, 'score': 53535.67465877533, 'total_duration': 55394.11697602272, 'accumulated_submission_time': 53535.67465877533, 'accumulated_eval_time': 1768.7811193466187, 'accumulated_logging_time': 2.64736270904541} +I0916 17:16:21.021446 139728305223424 logging_writer.py:48] [68464] accumulated_eval_time=1768.78, accumulated_logging_time=2.64736, accumulated_submission_time=53535.7, global_step=68464, preemption_count=0, score=53535.7, test/ctc_loss=0.299678, test/num_examples=2472, test/wer=0.0974956, total_duration=55394.1, train/ctc_loss=0.28892, train/wer=0.100022, validation/ctc_loss=0.544409, validation/num_examples=5348, validation/wer=0.154386 +I0916 17:16:46.292183 139728296830720 logging_writer.py:48] [68500] global_step=68500, grad_norm=0.5, loss=1.38254 +I0916 17:16:46.295595 139747222361280 submission.py:307] 68500) loss = 1.383, grad_norm = 0.500 +I0916 17:25:11.220497 139728305223424 logging_writer.py:48] [69000] global_step=69000, grad_norm=0.5, loss=1.41393 +I0916 17:25:11.224076 139747222361280 submission.py:307] 69000) loss = 1.414, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 17:29:51.242532 139728305223424 logging_writer.py:48] [69500] global_step=69500, grad_norm=0.5, loss=1.36929 +I0916 17:29:51.249254 139747222361280 submission.py:307] 69500) loss = 1.369, grad_norm = 0.500 +I0916 17:37:46.474694 139728296830720 logging_writer.py:48] [70000] global_step=70000, grad_norm=0.5, loss=1.46521 +I0916 17:37:46.478430 139747222361280 submission.py:307] 70000) loss = 1.465, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 17:40:28.990111 139747222361280 spec.py:333] Evaluating on the training split. +I0916 17:40:44.128985 139747222361280 spec.py:346] Evaluating on the validation split. +I0916 17:41:04.318440 139747222361280 spec.py:363] Evaluating on the test split. +I0916 17:41:14.901090 139747222361280 submission_runner.py:516] Time since start: 56888.06s, Step: 70255, {'train/ctc_loss': 0.2840348426492549, 'train/wer': 0.09836699355836405, 'validation/ctc_loss': 0.5450164997268686, 'validation/wer': 0.1535171148554048, 'validation/num_examples': 5348, 'test/ctc_loss': 0.29990137972611136, 'test/wer': 0.09648000324985274, 'test/num_examples': 2472, 'score': 54981.33536720276, 'total_duration': 56888.05970978737, 'accumulated_submission_time': 54981.33536720276, 'accumulated_eval_time': 1814.6919968128204, 'accumulated_logging_time': 2.7203445434570312} +I0916 17:41:14.968538 139728305223424 logging_writer.py:48] [70255] accumulated_eval_time=1814.69, accumulated_logging_time=2.72034, accumulated_submission_time=54981.3, global_step=70255, preemption_count=0, score=54981.3, test/ctc_loss=0.299901, test/num_examples=2472, test/wer=0.09648, total_duration=56888.1, train/ctc_loss=0.284035, train/wer=0.098367, validation/ctc_loss=0.545016, validation/num_examples=5348, validation/wer=0.153517 +I0916 17:43:32.861318 139728296830720 logging_writer.py:48] [70500] global_step=70500, grad_norm=0.5, loss=1.39522 +I0916 17:43:32.864827 139747222361280 submission.py:307] 70500) loss = 1.395, grad_norm = 0.500 +I0916 17:51:22.955078 139728305223424 logging_writer.py:48] [71000] global_step=71000, grad_norm=0.5, loss=1.40142 +I0916 17:51:22.958683 139747222361280 submission.py:307] 71000) loss = 1.401, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 17:56:36.823678 139728305223424 logging_writer.py:48] [71500] global_step=71500, grad_norm=0.5, loss=1.37381 +I0916 17:56:36.830467 139747222361280 submission.py:307] 71500) loss = 1.374, grad_norm = 0.500 +I0916 18:04:02.302602 139728296830720 logging_writer.py:48] [72000] global_step=72000, grad_norm=0.5, loss=1.3888 +I0916 18:04:02.306432 139747222361280 submission.py:307] 72000) loss = 1.389, grad_norm = 0.500 +I0916 18:05:22.048189 139728305223424 logging_writer.py:48] [72069] global_step=72069, preemption_count=0, score=56426.9 +I0916 18:05:25.127097 139747222361280 submission_runner.py:857] Final librispeech_deepspeech score: 56426.86305117607 diff --git a/logs/self_tuning/ademamix_golden/study_0/librispeech_deepspeech_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_0/librispeech_deepspeech_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..30e1473b --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/librispeech_deepspeech_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,40 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/ctc_loss,test/num_examples,test/wer,total_duration,train/ctc_loss,train/wer,validation/ctc_loss,validation/num_examples,validation/wer +72.46234583854675,0.0,34.61258101463318,1,0,34.61258101463318,31.05694238337957,2472,2.3705238356386977,107.63924741744997,31.939386414583453,2.4035197577988447,30.99260347598955,5348,2.256911118621156 +114.427020072937,0.0361976623535156,1480.9474048614502,1106,0,1480.9474048614502,6.257749290276792,2472,0.8995389271423638,1597.7229297161102,6.406972739363964,0.9413664109251888,6.318610137884268,5348,0.8966832424081495 +159.63744711875916,0.0753614902496337,2926.629233121872,2883,0,2926.629233121872,4.628748628943803,2472,0.864663944914996,3090.896448850632,4.940423060307439,0.9072633120790204,4.846255274261603,5348,0.8700429681842321 +206.4729919433593,0.134275197982788,4372.030340671539,4783,0,4372.030340671539,0.5881865362281438,2472,0.1848150630674547,4585.448388338089,0.6839214356055066,0.2145233570561233,0.9085786806811332,5348,0.2542075025346401 +251.97098755836487,0.1728487014770507,5817.834228515625,6654,0,5817.834228515625,0.4538967231756887,2472,0.146385554404566,6079.034798622131,0.498515827768839,0.1641555358319402,0.736931927883891,5348,0.213334620769565 +297.12858510017395,0.2118051052093505,7263.678015470505,8446,0,7263.678015470505,0.3912555346312665,2472,0.1276176548250157,7572.289734840393,0.4237485521826537,0.1401586722497808,0.672041145126959,5348,0.1918794959687153 +342.0227270126343,0.2510788440704345,8711.645572900772,10260,0,8711.645572900772,0.350934586505581,2472,0.1148010480775089,9067.41434621811,0.37034147229205,0.124100866326525,0.6151254756253768,5348,0.1764012938734128 +388.9419791698456,0.2914483547210693,10159.301994562147,12195,0,10159.301994562147,0.328308681205239,2472,0.1061483151544695,10564.632512331007,0.3353088900237093,0.1132540879612739,0.5805158808644766,5348,0.167498672331386 +434.5560202598572,0.4015920162200928,11605.590642929075,14124,0,11605.590642929075,0.3159095949738693,2472,0.1031218897893689,12058.94044137001,0.314595242569997,0.1074658723978894,0.5686893106728451,5348,0.1635784290059383 +481.9424600601196,0.461036205291748,13051.91927099228,16026,0,13051.91927099228,0.3044302253371185,2472,0.0988361464871123,13555.112949848177,0.2964337983923912,0.1024290902754711,0.5432022950101718,5348,0.1564235021484092 +527.1802635192871,0.5412375926971436,14497.74052286148,17854,0,14497.74052286148,0.2936940740128395,2472,0.094266041070014,15048.5044631958,0.2800245639965031,0.0967770039586384,0.536387646847373,5348,0.1541350842466084 +572.5648229122162,0.5980556011199951,15943.188425302504,19618,0,15943.188425302504,0.2908651364604168,2472,0.0940426136940669,16541.64181613922,0.2723956721104032,0.094386574389188,0.5301783712891802,5348,0.1519915029208709 +618.8605799674988,0.6618938446044922,17389.614331007004,21517,0,17389.614331007004,0.2821661881411704,2472,0.0907115146344931,18037.02626419068,0.2613459662660684,0.0916367636088407,0.5153653583829365,5348,0.1470380920195046 +665.1551597118378,0.762861967086792,18836.100885391235,23454,0,18836.100885391235,0.2779084285679721,2472,0.0894725082769687,19532.2314286232,0.2536218819839647,0.0889359593572521,0.515249051888688,5348,0.1471539612803553 +710.9896395206451,0.8335576057434082,20281.97596359253,25378,0,20281.97596359253,0.2664396442915672,2472,0.0866288871285519,21026.32065486908,0.2348187262523939,0.0827012398651775,0.4875852750778581,5348,0.1403756095205909 +757.0047874450684,0.9037110805511476,21727.520691156387,27251,0,21727.520691156387,0.2573770888444416,2472,0.0821806511892429,22520.22661948204,0.2219891192593298,0.0785247945809669,0.4785246999478852,5348,0.1367353835755322 +802.1866991519928,0.9844353199005128,23173.42985391617,29040,0,23173.42985391617,0.2483214390847796,2472,0.0787479942315113,24013.617489099503,0.2104059692594999,0.073662257894135,0.4613453330633413,5348,0.1321102689132429 +847.7015099525452,1.0710821151733398,24620.16385602951,30860,0,24620.16385602951,0.2418787603232466,2472,0.0772043141795137,25508.309183359142,0.1980789526051357,0.0695511546482692,0.454876955479581,5348,0.1291556027615507 +892.7218158245087,1.1452579498291016,26066.00140094757,32795,0,26066.00140094757,0.2354236815681656,2472,0.074360693031097,27001.54048967361,0.1867580999108773,0.0659845683886108,0.4414691859899287,5348,0.1253029498382658 +940.0010640621184,1.217942714691162,27511.845928907394,34725,0,27511.845928907394,0.2321519231079424,2472,0.0738732151199398,28497.04074501992,0.1802414469286712,0.0634688999123327,0.4364551055951125,5348,0.1239607975667455 +986.9447524547576,1.3041667938232422,28958.37071299553,36627,0,28958.37071299553,0.2286605688270211,2472,0.0720045497938374,29993.15261864662,0.1760754508430931,0.0619769233701245,0.4329456025059021,5348,0.1222710375126732 +1032.3531494140625,1.3909907341003418,30403.793350219727,38456,0,30403.793350219727,0.2286049956852055,2472,0.0719233034753112,31486.40464568138,0.1753784761435418,0.061573980800331,0.4328299337106941,5348,0.1221262009366098 +1077.666915655136,1.4649040699005127,31849.34828710556,40228,0,31849.34828710556,0.2285071567117233,2472,0.0721467308512583,32979.711805820465,0.1746056823748116,0.0610839155127444,0.432282542665009,5348,0.1220972336213972 +1123.758908510208,1.5458810329437256,33295.38259077072,42123,0,33295.38259077072,0.2278372798245048,2472,0.0724920277049946,34474.33810329437,0.1733037894971375,0.0612200447592962,0.4306325699781494,5348,0.1218365277844831 +1169.7595500946045,1.6316978931427002,34742.66511321068,44057,0,34742.66511321068,0.2299340884895799,2472,0.0735685414254666,35969.98772978783,0.1736082882334332,0.0619823685399866,0.433137451809574,5348,0.1230917781103654 +1216.3707485198977,1.7183234691619873,36188.63485765457,45979,0,36188.63485765457,0.2336581946819149,2472,0.07413726565515,37465.13506793976,0.176779414822418,0.0622818528824006,0.4403990288922795,5348,0.1245691111862115 +1261.9813685417175,1.7793095111846924,37634.08888673782,47856,0,37634.08888673782,0.2408211820117426,2472,0.0766152783701988,38958.67030453682,0.1866825997441976,0.0667033308104046,0.4603885392335995,5348,0.1301211799353063 +1307.3615272045135,1.8513941764831543,39079.88468909264,49651,0,39079.88468909264,0.2506045752951803,2472,0.0811244490484024,40452.2503619194,0.1944978369906489,0.0705367303933046,0.472085950842626,5348,0.1332979288369623 +1353.447606086731,1.956232547760009,40525.51788115501,51465,0,40525.51788115501,0.2555358410220014,2472,0.0831556070115572,41946.51808142662,0.2047566513494253,0.0740216391050318,0.4808824695160237,5348,0.1350263119779848 +1399.1608033180237,2.0289409160614014,41970.87526202202,53400,0,41970.87526202202,0.2687973568859281,2472,0.0860398513192371,43439.98548531532,0.2253668548049677,0.0806484108271757,0.4989853227188818,5348,0.1406942499879303 +1445.3778800964355,2.102470874786377,43416.44570946693,55331,0,43416.44570946693,0.2769799916526873,2472,0.0877663355879186,44934.14357638359,0.240866233795501,0.0856035154016629,0.5176834925030138,5348,0.1459180224979481 +1492.5967779159546,2.17545223236084,44862.44470882416,57234,0,44862.44470882416,0.2789501036357184,2472,0.0898584282899681,46430.06482982636,0.2501228837630154,0.089447805324287,0.5302517557326201,5348,0.1483512769758123 +1538.037742137909,2.2553555965423584,46307.86073827744,59070,0,46307.86073827744,0.2836022434915156,2472,0.0919505209920175,47923.331095695496,0.2561507256520068,0.0905967361651846,0.5335001455915963,5348,0.1492685752908801 +1583.370682001114,2.3385322093963623,47753.28298354149,60843,0,47753.28298354149,0.295044841602684,2472,0.0956675400645908,49416.46483445168,0.2732018576283178,0.0970492624517421,0.5415587974212628,5348,0.1533239994206537 +1630.123468875885,2.4058196544647217,49198.828130960464,62733,0,49198.828130960464,0.2982510716417188,2472,0.0948347652996973,50911.28756451607,0.2780718176932209,0.0973433016242941,0.5457853195009544,5348,0.153208130159803 +1676.6409544944763,2.5173280239105225,50644.459433078766,64663,0,50644.459433078766,0.2912591638976708,2472,0.0934942010440152,52405.86308503151,0.2776010644540219,0.0967062167504315,0.546718027928471,5348,0.154666151692174 +1723.1894779205322,2.5836801528930664,52089.85448694229,66583,0,52089.85448694229,0.2992008052938899,2472,0.0968049885239575,53900.25670194626,0.2903884168659453,0.1006757455798833,0.5558291774977396,5348,0.1578815236807801 +1768.7811193466189,2.64736270904541,53535.67465877533,68464,0,53535.67465877533,0.2996781042325311,2472,0.0974955822314301,55394.11697602272,0.2889195519547448,0.1000223251964345,0.5444093611864577,5348,0.1543861343117848 +1814.6919968128204,2.7203445434570312,54981.33536720276,70255,0,54981.33536720276,0.29990137972611136,2472,0.09648000324985274,56888.05970978737,0.2840348426492549,0.09836699355836405,0.5450164997268686,5348,0.1535171148554048 diff --git a/logs/self_tuning/ademamix_golden/study_0/librispeech_deepspeech_pytorch/trial_1/events.out.tfevents.1789523586.7e9feaa71658.38.0.v2 b/logs/self_tuning/ademamix_golden/study_0/librispeech_deepspeech_pytorch/trial_1/events.out.tfevents.1789523586.7e9feaa71658.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..e2906b9d1b7129fe3c5ab19fdb0b72c05ae6ff3f GIT binary patch literal 93662 zcmbT9cU({3AID1>MHG=;gtAf+8Br86Gcuz>W~60gRaPm=N|dCMB9$bSQ4&H38BvrO zLS&16ziz(wp4UD1e)`0}-|zeNIL~v=eLp{UHc+POU*2uwR|+yR?xAarJTz&kG|S0$ z!4e&_Wp+zk`t=;&L9xAy-4bU9r@8hH%XMZuIN9mAx-78Qv8(*=&eqGF7P#0sWf#>_ z?e;hAb8NPm!rkRo(Uoak+9|3l%G*0QI}cW@C)-%2z5Gepx`Rj5S5#1}>pa`m-qy+4 z!P({?lShfCdZ70PJiVE|bZ0A@nad2AvUqD4g#XtxsRr=_r|2Lq)VU0-& z5s&`{Xe)M7tUu4*VV14EwX=)e(*GhV|7VD}-_$(F@NY!;$~BV}E=7c%t{lIzY?rQz zofR9pRL)6{*)FrKNtS~36(*4LMg6Wl48803HQH{u1;=0({ZiVgQrS^90qRVI;HUr8>r+Yr^^EdES z@ztpc-Z^Yw7e#GFB^Ntq7izvkT)rb;kxhhC&%ZGpT1+ZYNWIU-bX3$-tjEL5rymW2md?X3>5Vho0D<& z4)f+MSTYY1D{Emx8E>8wV(>Td-o+U&6~5MI6UEzr`ydF(JAC6wzrP`kcfWb6P~4ae z>8{vSQF*D8o!#Q4mB*g-YzNmRF8?h8HJlUHx9;owO}cn(@=t~A)qnm2q*~&cRC%19 zHk~!^LL1q;fvuLZDSo-v&JgCK+=q>jdOpe?tXTvU#@}?#Fm16G-Pi}l)Q zamn?15m5O3VI#yT+>t_V+9PwIh+e;^ilR165th%R2yWsu8rH(ohpm>9!eGGIpFk0} zlZ_Ck2t|rqzo2G7@$+_S6-6DI!a~&F9nKKkJxHqivKqfY*c#+LbHIP%QH4Y zoFW}5?j?Bi1q!XX1yvMvX^NajvbdwVx@|cJ6rUHf)xs3iaZ%*cN((54En_3ZDGHH7 z?bom}plH-p<}A~r>d_Ph)xs3i(fw?y%rBsLvz(0(r%*#qYG=={j8C(lRvuUzG5p<2M7N(%qLSy^P?m%(+4jUm(p@9_bM%6=K*r+F0QPig? zI^5w=)cNB394IDhvDH#iyqWg8Hc%Yu!A6Kv=px0BIzNZPTAW|{zKWs&P4VC+kHSpv z4fd!BE7@u(DZb>iwgQTL4>m%a!Wbz^=1yD)6lMJnoMVn|C7Qx>{x7U<(PE<}ti`?| zY_*gWGn>r)02ErI*a&e73miw|z$Wd1qQS2GDvE|Q#qFs-usU|r+5SLb7R*))Q&30u zL#1QqfuctQ8zD|%gX0KnuGkhRR&KAK#;ipnnj*E&2dr+jaQIrFu-?a33sX=>_n5&& z=|FKTjExYdSc>CtvrxxhAeqZ2R8c6?6slUEu=?QI?^l3g_d2#(N(!&HUaf$_a5Ebr zPT`K@7;$}UI;_Rrm9pu~TBrz6G%LaCM@_!x0L4cSwpy5ia`&O}ce8=wwl^CgPT_^) zxUgi66;QbEpHM~7n5OXA@fxemTvwU_#nnS>wJ-&>qYkvZstgp5kFgQr6#h7l#`O~H zfud#pvMP!uG)2O^BCH;-Y~%$LHqY5=VG3$3o{ql#87P{Uuo2=EfjEu>t?HTqh4z}; zRTQc;MZY(Xu=l*B{1K3scDQf0^6gKdT88KgO^T;uKLhj)*6RynrHacWf0!Gn!(Z zVIEemP%rHQYjJu%TP-C;x8Tz`Kw*1`jS#1Z$8ii8>iQNad=}rTqEMqLG^XWX_2YZX z@wj;7%2o?gP{&0ppGozBV#HcDLYyKQ$MLb)!x1QUW*KELJE}QN@oL~rtR9mvOBdE+ zXbxK~OhFwNV~hO)fnw4FHbR^t9mgTFX7CoE&9pfCtxtA#13<6^B^9VMW+6vjq~QxxJj zJT@oem7LsRrkTu+YC}^z$vJ`5rxg0`28wkSY_*gWw{#y20E#J7*a&foVjPEaWQzx| z7BX_nswmph6w7}c!|Gc@Y_x$QQ<1F}rl5Azxc5!`fTFlD8zD|nisP91VC71n(D&V2 zMWIepj1A^F^X$jzr+{MLF1A{j;y3?l^Nab@W&y?CU^YUW;yY3dT%x@ID4LjlucFYP zDNH6F!s>`AnG1j-VI*5EOhFwNR?QQ&f#T`}HbR_&cm6*>`*{zb=;IrDkvT5f(G+K2 z@P35_dN-I36diZ5)lyQpyuDrl6k)+^gg6E7KFab{eqVrM;Gi#66zyq>qV_?!7FUWo zBm>3Ip=`A<1$Ew?_EN0HkQ{6jlfn zs`AbMs}`mx=N}i_T$>#LitPGqgg8Y5q*(v>d?~EO@PhGGT>C-*PmKr!npTP-C;MRUJ3K(Xu(8zD~70Vy`x zlrIB{dr#A1K@-aazxN0>xE*wpy5iIxfZ(mRtaexBb}&af)t8@kg$|2T=4Hp>l=U zQQc^Y;v@^CxbjftFi_kc&sGal{Nh_L?CZMzKoKyFjS#2MMT+!IJ6Zxo>*J|a6y0fx zV^(HJaqdTK5>V_-VXK8HsI{1^@fPpG%s9_Rh*R`Jiqgxnc&E6%m-gS99;9a=@9Pg8utrn)B+#T^OYz(YLo2zVuIE4jL)S0zy zEl|9#>zd6R7X~y%ufa``Vvk=WUXO}x#8yj5k#r&OF;FyX&PIq+Ohk%<<5fEWMO543 zDvI7TMQESKNbx1!#Rk@5OeeNln1VXGcP(y$XGxYiY=k()6r?D2Kl27CoEz7_&a6cr zn&O4A5>n_#e_RO^H(IjQQc|3ppS%^;;!S%tLY%@1Dds+Xu>>f5^m|oN7}6A>H|rsV zRmQ{#Kry60TP;jM9T&$+bJ5+Chp-Xi6gEf^p{RxwIf*Z;DEiVAIuGk2#bF=A`asd^ z99u0VMav~O@P4=3B{o8wVh&R53+kN=Ycav7?+s>0^`j}~UzJ0OK}7@ffufNaTP;jM zUCH@msCEx_l=^5kLY!hgQoM6p+Z`xAR7|g;=ucA&SS^DT?-H`CfWo7$)qmAeQnWi1 zgLfyL8?h1M6!u8b+2rbapwRlUp=ulhXo`gF3ViDDk>a#Kpja!r@xN+eil6-Ri>$GO z@psf11vWyQVkuJe(5vGL6m6sC++=o?5lvwn^$RI<9~Z0wihA*EwJ-&Bmb7u^2@BX! zX~}GaIE4#R=s)|}8YpBBU9O@SNK>5Y{uwFmWzFjV6lY`DYGDd$El!(HeF+rViEM;8 z#R{bOpm^vGP%Iz)rix+^P2qXF6e$u%j|&5eiI!}&Fa@<1`?MBD0ma}MY=k()YNRmw z?mZhQYJF^w!|W(ynj%*9HB#7hl(Peho8Q@LVG3$33_p$C0~Ap*O8*T&oWdO`4t1M& z11Rb$o~Q~DcqL87n@6$ORRMccoyKgnloX?4JGlVG+ZJquIE5!tIL?b)0TdyFTHa#T zVlcfHUM9u37RH~i;WL~shOyPc6hHVqs`lkDd!V>8mW>dn*nkvPQ$3@B!twHeDvBXA z#W=60NZ}Ly!4Y;;-5j=BnBqI1!lOldC!jcZpN$Zw*n||8Gj=8cMXfv5RTQQ)MMCrw zr0{wb&=e@5^Vw=)if??13zm2BYIBztY=k(47gD6%vmFi;kwHqgnH^-fIt%dEV zCMiJiB%G}lrl8iM;iXZRVJ+Ikuo2=ETad!zWPiMRV4v+gm zmdjQPQ&4L$A*B}HzZj6uMu=1RBE{wzI%i-lb}f#pq8LU~crJc~6qczE@m%kUD_bok zh1s@qgJCW9tzjd?DYhZSr1;Tz7iQ3u+f@|9X$p_A50D~r$;N0{i`KSmwJ^nJ{&A5U zvMmTG?#yQ+#3}rdqI>SAU9c8=oMzl%jtg^|qTDnODJ<^4d_TF>#yzk)AYElfcj-M6Zzg#Ko$g(<%BkBcK?zIFkMHMQga8-O^)KBO4+^}&3g zh{-a&%j~EzG==RZ-dd==nVtv~A-C9SVTv+7MgLn?_~wE`9vdM}5sVbq+T1DtirYu# zRZ)zkDMriX;#%w)8Xg4{-U)2AFa@<1TmPI+g|!%WnvD>r2tkUk8@w9*N1@#N$ak*6kpvcu`Bg82}k)n0hsyaXseNO2fv!lk*6qY47 zaV?U!zIz80j+fYKVG3$3^2!wU14Y0MHbR^t3@J)Rn$HA^U84i5D8|zi@^L(hgzJab z1BI0(TP-EUsN;`^1I1S>HbR^t0x4eVFH-;ttp}-96ccEQ8=*IFElg7U@qN9Pg>1Dj z1@#N$xP{wMpzwIjMu<~HB8B{@NPPdo+U$81#YCE-g)?t0mNq(-2W!!5G+Ql9L0NBl zu96H;+?&Woh*Lx%MdAGG_%`mN%*pqe6~!c)V$z^&r0{H}fmd?spJJHDB9W{-nD30Y(%t>8??ohG z_rl*%=hWD0VG8QFa5wFr0TiklY=k()8KgMxxN!tfq;<}%qOhhZ^w#p$BKB^|0a%OL zJ=tnuijVxWq+>F-Hvq-XzHEdz#W|$-+Sc$aP~;B(Rz+b$Q%u>+TMHS7`vyR<#Dc9B zrl59|THGF2pm;ikjS#0uM~VXT!{dP>%QWy2b6nWc6q{~e#2saOr%WFy2WGLXV~{qK=Lp&8yHpP`sVQ>^aDqlo))6MunpIm%WGQ&2m~OsO^g zGVhznMu<~nB1QZ3!;^qw&WZt56tiiHSEZS_7Fq3btzj*6J=khtiW2^D;b>rs_gH&) zvk~GHmylxChcnlJqEywoiee5;5r3RVp}BMZ51pjqC&f3ZkHGUTOnAgjQEw~s@l0K2eGj_7?2L=7HT!uKv&mr)sxe%WRIRLUb!%byfB}lQr*O#W58zV?k{3cZZ znQ)CO7Y3nD__oxo@Bk3&J6wi1h?^;ZG#|D?ukYXK@TuG9qk^tZCtU{Qc0Ebo&Mc|T zJNbJTcghZsQVh8u28sH3=A7S4@w-^6^*<^N{5u%ttEQbzxQyz)p4qS`?+0sBw~StZ zR%p(pN%(wby{&dQtY*`-d%))oOUH9T|MdOLu5Que;bW0O)3`+O4`{9$#QPn%yGh

z{)Xnbwkfz7Uv==;f)xv2$<<5vh-Uj!mocbtu71GKzXNeHNI9fn?!g5~`xQ-Ucy})N z>KXq{)mHnwTfx$sORfGhnqd#0V>evZQ!8`G-@M}+JBBJO-p=Jo_Z>}BRUHGTi(MtHLYMoFnrDk8!OrL7$1dXy`<53Hl{*o{7Ij(VMwXji! zK7BnEx{6FIu3WfLQorbGbzXNA<|Iy&%MhIt_tQ8hDTimw2arxr?S3;LIRf8L$}(?; zAdah+yaSMzZ@6+{5bDHpRsI)807)(5GDJaSej><1mtpGxBwgd(DECTrWeR-ormUbI zg2iVP> zf=I=|Tdo4qLq9G^SX~PS&&M|UyMNhSP*>g_E~7eiZBh-w5m~IScn@fnpi&+)lS*A0llP&zoR67BLQ}Z& zm5UT@3c9^;`usn?b+g;HRfw8ULf>ac_J%o`ki%t&&WU?}oRhAPkK6)~>F?uPdQ^7&7=cgLbPO!V!>qrf zdM<*LbLCQkEa+q@3m|%m$Jrr>f}~GFkOLbYw*!zXj?26lkWhhdjQF&Eiy*gtZVUpD zyj5Jelpt#!^eG3Bq>WsLD2R+3f+XB>n*$)(xnYkPkX-^Fg}DFaHG;gGpnnrUN}h1# zQi2TJX%Gq^E^oLDQIJqy1nJ$RY#y||cDqztc~)+%xQ>z>?AEjgB7I4?n+HfYL%1LY zi8_6hiaCXt|P2k{JX;_#|fP-~jaY52Lm>S;A zi>D5PgXKPOsnt8U*~&5K;O`lm&B4L>U%5P~9K3X$0{U_!D$liurhA2qkQ3RQ8xy1!QwkFD|tgr0&r2=0Quxt4cLBdnlqPl#~apl4w z)Oql!q2usZQQ~DTLlngQG=kXOeW3&(bM>z|Fd)wbUjKI5br(U7Id|(0D>7jqS1t@f ztw`3^MeeX7?}l?3q97XA5Tq#chzzv7ewuFn49FjSNe*6a#QRH3O^f%b1Cm;IE{H** z96YDFHg=G{Lu$Ke4u$&QPT`U0Ac5tk;x!E;=u;jqqKD7^>{ZZ(t( z`X>i}xvet@92`BGOB8pomR&K9to_&3^59@wOD;#y!OdSb#fVqw^+LeG^=EJqHFU6d zQTIOJ;K(^#7VY4e8z*ryUL78M9#(9LJy$Qm!Nb0v#h^NE75adKH#u`b(mI$L-l^+r zT7!c(uHsUwcks^dtI)w~&O|DLgD0)$@}zRGafL3fW`NAZUeFZWe7H!_rcinlr%%RP zGXNayxPwcnnS-BPa~Yx_Hg6DQ%Bb@XVMQ7))Y->?qzk-p`*PGp1Q~km`cVLR<;;}}gHX3oXN{3- z03dQ}xC~JcnV$&qIYMt4bc?~uuGZdIxgvi|B&lo1mj`&zM9R;o14wl|xFBJ5o!{69 zGt{qSq=ULfd2t!lsq2|yH=N`_H_mAST04I(O@g`_tv1DBeetUB0(Fhp#|8b9x=yBS z$^vx-hH;7F>N+$&3rBYC+ZlZ0NaYBZBdD%kv9&PbM#=o?psu*%Ttp4kWl{T2F{tZu z3YSHzYw_==xETXRK6?Wz7I>bkm!Ph3VI>%JchFN8P*-vm7bLB^sNo$>eQ^%dWuL>P zR9|TEqyixS2K9(p03aol!~}W(WdZSZYbQt+;h+h)iuAE zOR1T<9{l>?4UJN{UZ@h&UlIjg!B%XNj>mB1n=nV{Dx+1na^Xfvb$pkj{##&9uD0MZ zMCT+&9_QrQ$i;ZU@b@Ua`3y*uz&o>hp79*KRn`#Cx_VCL%7sCw^Waz$nH&JwXw7Aa zf`m3ekk*suB*TiRA27SafCLM?X*16;W<2z7U0mCvU70CFRW%Mb-I zZjB%(i(h?*wzt<~$0`P7mxUw;&)t6(kAAN@vATd16&$?o92ZeT2d5UFFb4+@y2NGC4&L3i z9&X0S;*a<^+Rz(Zy#xnG9aF-fG0QDGgM&NV<$|PjFg3hi`%^xFgKy?@snt99b*-c5 zV6W-(7J!3yKIihJa`61E?zka*I?mV$P2oie7b)5lEHrTXI`mp?2B(jjpShHpIauxO zh33#GSD##Wkm)ZQ1YT3Tb}R#r;T0nSOrfjXOykOh8zptYu;9X)wlF8>uW}ipbE4b< z=cL3caVvoAQoH<_0r@WQcHBTc9!N}sCdmNuL4zxo62zyU=V$=w+nLJ{1u5)`Am4&U zc>qXP-WXAK|M{Fd&5j@1GPdxqu*r>iH`HCdKd%p> zvMzo8kpT&}mgL}|V@L7mPm5fl4M?TGxF80Ja&X4SSLlP@uQx9R2M?2f!49T62ge7i z;w0xi`#lWMYBk`}BsloehOUTqzVS_MaPXm~T+lx`c*C&s9^l}KZMZ~n2RCURg(FL~ z(eeWakLbwd2s(JgoIITKxjQDCfrF=Y=OSw8;FDiV*MWnN8E{#&gALBM#m%@Q)F01- z^NhH92@dvZ+!2G0-g)f>4xVSq1xf2*YIwu!{y2bx6-RQZ)jN3SiVSqHp~iqq;NYkU zT%J@8PBz?)8)9YO@N>|n%cgOWqD{efAWolq%LzBY!Jllol$tsCK+EuJ&?uYzn%K*$ zvg79pypW`PA`Xw?fSoo5&{c92E!lD@8|9cgV~@d{+;7fhh|Y<{5S$am)KO;vWbW8_ z4+bR5j(!iuT;VK&oHdJ00g%M0T)8j^^<$!F>&jaIVm*t?5Czc~i6EW3I-rAZ4f^_kb1Ym~eNHcjeZK zUrd9z=O-XY)5*RwVMTOLbLCQkG@h-Y3?M$4T!tt}`ZNT&=ep7cM7409>pTV|W{D&R zA8dROkN*BAbJn?k%5PT!#Y^4Z{E^9Ni?%^d8k-DC? z#4Yvm2av5FxN>0->hy6(M6U?|68N3V5CySWh#=e5k0k;~J*W6h42bVa0g%H-aYeji zNvQ7s_U{PI0gov^vdm)SW_TFCo{w{Ch-EW>MBwFT;XIRep$V!r! zt+gK>K7|&I?*o;#4Ur^lu{!#fu%l{34H`D+?{LQ1EwfelF^>qTPGcu56mX`KO`?th zUDQ${PJ+h*?zTZXw~aQ0FKSiDhz)S1$q-f zf-2kA@H|d@t%c(j82z1i8}pAP3R|`l88viScK;dA_5OzKU%&pf!lb=K7%j7}n|MFk z*O|PFhILB{B?2VatakfFxWAQ*J5C(-cPObBeg!G?I7}o-t21hV9x*L6O#a5oL}wgT z*bzs>R_`c7w4s0Qs)7{QlMz@jA6<27hqX9GagxjUd>($EM1!t^};Na z(iG#%8Uo9vHbjUhimrDd>hydur7q6&s^QtSK_X`Y&0|+*6n6K5gW-HTJGyEyR)=4RC-dNYI&^nb} zh!D|vF%HIg8RdR!9I$BW)pOWZskB|80xY3EynB65hkTt3Ea&?X^};Na28Qn5xEWa9 z4g58&cO1qTl9H`#VbO9rO1!>QzN~Z_A6kCG9>DSS*V{)gM0MED*^9^ z5Fw&0$`Qzt=j`VNEUOwDclWDYm-52`EJHncKQ**eMo$11$2LU0Fbj3|+*kdr8>~xa zM4MjE(JX=d`u)st7mF}lb2OQgPuDV6S39n zS@l#))boAK?K?rwxgUr?sq|dv*co^K;P4g?phYbIMr4Y%h?Y^f7`KDMpMak2{tz)W z({sM%UM*;yDihrNnRZipRG@XbH051*+_i1DGc+0dnMA#C>!dW0>a?O8%uDfnB1Cjv z3XkHv+*)eT23U>;H2TJ{JdYD#>7LDd%1dj)=NG`z_#ja)%tHOtSP?5f5LgyO6Ct83 zS#ii>)u_>1U^yMQY`lM^(jJ@;U}>NkgImqv*!n@hvN?jN7iOW9R%hm;Z@@A(jtCKD zi9dlX`fcuwf#%URBkKgi5^_qS(o!FKU}N3-wF!PF{?m0LNknNKlCZPhR&IR>lva3` z2&qnKGhXqORuVLP9?)$rAmSt_ZF3w?X^KGsA3$lhUJ^@&B!bi#W^`CW)i(H1e_6fQ>N?Q0)`(o*$^n3^fAzhOZ( zw9a91U(=X&ll}j8(@s%cQQqFc*?F*HJ=w-G?d3ns6ZOKavx5K4 zd(U+~cs9`fJP{%~FM+&y8F)a?5?BtFMQUuX)Ij=a0hV8Fc`Wxd24(|`V+Bz!%t9&6 zB z&8(x*65Ws09twK)@*qO0)AQwsA2_DT$2Ix`-Fj~#PJ*68&+$Hwk+7%&Klz%tgGl-( zJ|A#8$88Nojhh=U2xctpGi{+#~{}(sR8zme@1HzIir)7V-KXkty0DOfTVLJXq8e zzfy9tfQYG?o_i)W8w(ncSAR5&X*a=_1zKl??GZ%!ec-}wXfl(uiF)DINok&VlJ|p%JccekFuqDa)0p=eFZ!qa+Ru_vipe&U8&8iJ_wO9DFMj zE#cX-O&I9;MHeEZIz8*je!(%h7I%>Yx=THYI0<^Tl5c=X$?>-@z}a)t0YuV2={fEG z`9q-R4nv4YaXr7kv>wNnp3-Xu==t&pB1llr6URj1#J^kDwm;~(&jcc)hI&>_34afI zUO9~jqxD=rEgE;_nw55VA%BA{5g$&nJ?k)iX7~hnBvopyxdf zL~QkX9%F2Tdd`g;ehKv4b2$+xm7asA&O?_y>9A=$w1~*HM5btqc$tff@#?wiFF1Sl z-$=yNP|rP-_JRi9&0cYvX*VVJ1X`z4SSTVjOdf)N9Q3vWQ7_y&|NoF0hG-;jae;a9 zUQ2|C&dY;)I4|QHF0%)g!OP|h-c_lz!iNGZ0j~Ic>|jNrQSrnFbk!$osSNV1HJ(S7lgA|Ibp@2%T&fEa@@ZWwteW-RX$SWR!%`8za>7@=Zh% zLq+M?C_ex#VPaKu67)Q58xc~So?Q)iFEAY5tkn(Zj_~*FLOp06q6VOhk(7`RA~;IJQe~R@sA|$Hoysf_gqU^dL@ri_^_!fu1`g5g9eq zvz5|DbI`NqIUO$j~0J%2NT1d+Cs*r`G5oOy<*7jB)@S(=M~ z(^W7p?Jp7`qVuxiC2rD2dsC8tMgB-xF2fT4Mu5d^(>`SJRr+!gShk-e>V;V-rEUDP zv;tTzr4b>bEH-bDMZJ5$d|+`Y`Jf(9skDgq0xav&czRaQOeqJJncs+dVHRp#9KHA8 zetlR$gov^Xe~&C@-1Xi;^O#n4-J4_u!40Q6rG>rVeQ*AB`(hQKd#6OiNl@BM6M3ZTqS3j`pZW&Hq z4NAM!mWUKrno8Vq9NSZaR;i%0rJaZ%L8Xmq6M_?eNKU^Dl$M}FWYkb;^-t&D2Bn$y zCc|B@<9-9Zl6_L22?6i9o58X4t|TduGCpI`|Sn#dIQ5v_&-kh>Icj=LUYROKmn0 zQ!}OI40Tq4*4e{gc^K1f0zV0~&V!mkh$NR-{|mIvx`T*%;nqoM;QZN-iJ$?^5k!dS zyy$+yd0E@Z2sLo8&+C$^*7;R{rLW3%WYOx?>?5!=G9~JzWVzV=@^4@X98H9XvNZpS zEYBa6)PZ%Ga@ECdPo>h_e+sZ@#_mOyQ4u580LzQJM7=Nzb(U77zsw$31{D$^qAW5$ zk!81Oi9WD2Q(GFtuzdb4z_O=^cgERvbieMvagaJIiJ_v-IN!G4jF#xCsAdm(UY$aO zRHtX-dQWjoHhpwlpv^BlPsB;k^P;gbnAE=SXnD}{>NCXM$dC`bKocM{l=kWbnzo$e-4fQN*ySV`LocNjuqxCEs!MlCb ztbX|~ShwvTi2w;nAr6*|wnPCEtl8Y4uDEFyv>g-k@g{nKs;EQ@x&h>R6+m ztv%YsfSyC@5rI$HphpdVPNfntwZ^y5^c$MEz#f6uP~Xfk<;M7?n9r1X4YWd25&m(OWLi0Hf& z^XBDa<4%#l5}T^Liebs$D8SbcK$LzG0hQnC|JRqP~^7%EE7<-a`865W*=3AC*$Cpe~Oc9`t-YhDiD+JzLt%D+N7gClZn3dLEFn z7%5Hr_xJ;P);dcB3F`U&h&{;HKEdfAoINkTNMzJd&ngOE^+3;A*NHG%&&NOTZiT5C z4Y>j9_UA4UAW6?*yqh<&w=>3ro_FOFNz&?>8lbhT|9a4K&@&>odOfSiEJ8h-nD6Tc zdhT381WKi6-xoZWb>6*V2egPqpNUM-7ICO9wurlFW7R>=mcNLYn(2AQTv;<{owb5i zW-#q0dAmUCw7J4-ohiec4TUDN}eP1z&B zGQwuTT81TbuKh#>;XaS1L>-&&l zKsVwJ5hp>wjlzg(sSv1-Xo~0#wA+N?sjw}GDTZNlSbGg+!r`p2R*;;M#R)i&!wh4 zXF}_IEVpe$P^Hqm4hpnRzpXr_^>eW~4z2Tr5>YSQI;k_8KCj+Qgn3a=BSJ*y#Y-9I zB`K~Ee$(5z+o637OL>FAN9tpufQ8AN2%P-$N^XC#8sdW|5$Xr%>geuuXyGFM*u z0_!F-fe4Ur#(A^+3noSCwLb$&`!Y=6K+II&6M_3c~~B_&MPjr z<}&T(`TyGu^?t4FG2Vs833tO}pvgF_CF+G+C#8X*Ix|uA&o>hxqVqC`H!or1R$Bmz zqx!x(49kOf0T#{qJQjPsM+v|Zt4-7kvruQ8>Oamr0u89^5h0>1Q<~!%AK%miFQS^9 zKGbc0rP8t!1z1il;py4e?_D&ogkB=*g;}U|(fwSE_c$YO5h0>1qgo=1)7Zu`urAK` zTWn`o(vt;P99#0PYTTWjvw*7V6@*rCqZ$SeKV?h!9a0pXoU4U zMcrjsB2p#l`R=uzsArAwF;PI3p-Ut&RMZ(~JAF6Q^Vi5Nj-cnoeTk6j^sG6Nr{}s4 zRN{fo-h_ygpyxIXd3x^Ey6;QSbKwXg>7VqxuaV7Y(6h>TB2rw>S-%(H%p2`H&Xr=XdG;(KqlS9Eej>IA^lZ402&46!aQiz-)!c4T8(235M&9f=T87Hth=@m$}m1+W}fOG#r`f-?nJ z)~@8O%i`FxGl6AId!k;Lh5D&s5`8ZgSVrp*A)+j;+apV6ejRbR;-4kJ zawCs-RpWv3N_?@ZmNro@%tD<#Z#=ykpF*E#K!k|0D0e`XQG5h7>X0C-@IU<(`kf7(uM|jU8 zJbM?H2zu`Nh)9xF&(r|JRNd`B&rwf_*y{CczQPyveC_uwykD#QmI#ze&r6Svr7sUodap zq)E_ZOg9qs!mX3iK$ri2bQUzw)Sn0uofr8|*gA{KW+VVhSip>8hUNQB0hX>ucxTU3 z@)jxs%l=TJUYLb?S2*o{?oeQP8%>0WvV7MK`NtR{>7Vr6No$22=s9W<5hmV_6#ZI%%M67>ArgZFI3yr*~WgPvck zCX%GpGc~{|3wO^0JrCGO#8$88F@`%(&+#GC@O#Kjw-SL;>Dld34(fS$*kxI0cUyK5 znW8PCMK^2_XD;-X2R)w(B4TQ$=kk)dH=uQ{^G;PiSgC>BM*^+$#}(eAE<5h*$CqC6 zb`kZ$t&=)?ez!^&|ES!B14M}EyySMrc~J>6!He4&8&w?{maHcNEDP@OUf_A>`vhOC zI^a*#3$svqF8V$EE39$cULr)4C94OrSl`@x8P;Wo`inw_<;*hy7OPeJajW@jXX0Pr z(e6sr3$sucx8qBE?*hx(-b9Ef%NadnS<$ZdIB5FrWwL$3D)(5>D~U=wQriF`p*i9wyvhNkW!0(04W~M#{aRIoV^SIvW(Ra?jfgl2N>giFfdeyi zI428AJK3B_`X{AD*J)J&N}HfTM2ah|xv2wEMtq#=24|dU+C-3`(uUpTJt3TVwbm+7 zTCpyXQA4HKlsPMc(wg=q!f2(1TPfkLJatSl4%Y40AR<75(ln>?o~@%{X$6SRL@qQ+e zDcT}}^l>qsHaz43N*g_wh^d*-f`uO<^`4!Ae}(h= z5~5zp)_G%v8vfzHE6a%x(RqpI&CBf34PGM4eb0D?CE~3B%cVBF2e@_4Sm7VYd-jy5 zmy*Rmb59#s;|}kM5K)#016*VO*`JpKi&MFMIl~h8L4c)6B=3T78?&=Np!vtuS{^D@1j!%EZcJ?FAc9$ns1o^OZIji%aNkSCBS0&h^QB4 zp{{DQ^wYw1(RoRPh_d+hMHa=kofDxEo_I4ShG8lFB2mvNoq6xqe(xHa2vp;~5=jgd zb;g+yFo$`{w8wHgr~jTb zT{EyYCUuBQxCwgR-iS#0Cq4Vmw`>7=wp1e`#r2%KcNvZ?*F@e2^gK+R2oluu%Hm+0 z_&#k8-U2=$sONX<23-X`N9zz_w4T*!HNjok*wg1eteZt2B0z$kV{~}WiRgU$ z=>~eX8Av2at7mF}uRkf^_mEA@h}i1&JTGlO>iO#QeaWC_tuaKPRC->R{tR9A%(f9e zaoK+okty0DO8enrWQ;Ce1A3k?gNUh_o^QFmY6-2gLLvJy({9|q3AE13vw2@Z+K`pK z9hyu@W1?Ql*4g>&fTo}U+tx&g=)Ac1$9ef!x3e*@lvrq(L{w_v^G^X5%?7+b$v$mj zd?v6=wj%0TaKynZ-twJRg% z;gj3yXNh`Y7V3<1(A7{pd;WZp2oYtOKL}Z#Y-(Tt>oWMpXW2uQ>yqy&NB;zr(dXTC zXkW5n1+2@>JfdEhg<6-CIJs@G$K;UIvf8p3= z7)nB+yBei+~iADTym(RzM*m?u?b=j=4tm3|IHfCN1o%k!Rjv7h*7 z8|eAgG9pP@JyQex)91?xID0;_mWZuh&pQ@{qn^+CkEjiLR@h7gN~P!3KTEJ@c5t+* z4=o~o8<8p6BFcCzqQ|kB-$BnCcM~x+)AL+2qkhmj53f|2%(R=FO#-d+i7QV7%~thl z0j+cHdZJ#qby8yl4D^qLAv(=;bSs#99O-d}M{kGFof26Usw5pfcfHpx^G zlX6R&%>kv^Pa~54NojF+3kyJL+OvpAais;`^~AAV98h)ul-6(|5hSRzHKUK9(o}ou zECZ#zawIZps5I>t$MB0zBix8ET4@_c^F&mv@<jB)xTP?}FP5mPgzd8DrUg{ODjzd$G`OA zp-`8r7jB)@naxx4hej|jJ(?0BqVv*?H!riAKE)SBmfsJ4$*_b52(bL#%R761X_ttf zzgSR2)C;pv8rX6x1uvo=enW(avS^INHTD>H0iSlhWbk(8;Yy|L+9$x`oXflS?EWps z7@Gh5K}5YU3$-qJcVvoSU2=yLA)+j*qmjj8{|CGh&|<0gVTQ#kM1aM_l=mw4m3yiU zfhBJhQ7cbbxy=qDP%1rleJE=TUrN0G{4Abv>g^*k zMO#Fm1-1y60S%Qw&*}$>n40Oiu$P4+==uHN3))P(DgS@Fq4fM-H3ENe#MJwRANKPd zMbrzoPU^nk#%I&pf(G7N5+S1VQqG%~KQA*pfJNnS-Q^5R=^+7@Sxb0l&*RL0>Rw3J-enCK*`H zJtXRdS*S{IN&lqh=t5uo3q04Ih)8igkI&hPV=LMdy$bX^-;D?o)bqV_Cvf5$j_Ym;dYR4Sr$C-mwJv{J0qb^mI}sp3&mA}M9)0Y1qH6}|d3GR?B(0vQ z0p5@8cNfl{ryU?-tJib$m_*cbMRH?((6iEEB2X$lKiyUb^{jru*c95`oH!y=v_)u6 z!4~nSRHrTId36#IQ!_nB4RzQ9t<$}MYdq6#vX2Y2&ieX1Js;{b2{mA>M$`+pPUJeX~ofTklyUx3h9y(K216Z2W*XQbmS*Sgx z8zYkfEDnLL(lE#^QY)HyovJHxUgO`@J#Df8|c&h@+68>y0rB!-I8^8(#d zXo&+`M-Ko!C!Hlis?+nXb9GVAea_sk4|Ml3i8u**p8Hx26&QAWMKjQ|_YETHpY-ez z{%RJSJty5GBE|LWdwUy>?Rr^4Cg^$HVm6^Q9IJfNcxCVN@T23TMt!MrK*PVh;`WXDaHsXGPp?(DUmCM4(i94(?YU^*qA*xh1rS=S_)B(H6188e2r)h^szOzwBIQfa~G1zP9toG3h?GrWGUgC;Y01yL_$>)g@9;XTYt zp*s;GIxoRCI4|28ugC|MCZ)lF42%Co0hab}cxP#{S^oIP{z88e^};OFm4M0Lc8mcQ zy*m53L5Q;W&q9`won{pPOT31`SB7Q76#X|Aop^ub{E@Qma-b^wLL@O%l%BJ;q@X3{ zUaB=2^xXCj5mKF=m+!2HGktf)%Vt3LvbH~uPWWl(g1qLaz|h+J<3Z2)4T+?G(zEWF zE$u+hE!BufaXnk~+Kyuz*VS`1==o(^B1llr+bquD#Gmr`|9fHozeH2{&&K>?i9&WK zBBO?ScIoW;4fK3ahX|wfod2vB?#h=-UBhAB`u8CMBzewt77qn53hg^}0>%33?73O$17%=d{VnsOJWL@9}FtohA{PqAlVa zuSJxMb{_zG-a3PbshOTF)qmihmmlBRGLvaHB{>4EGkM`5JfPR~{D#kLD(MsT!mX3K zAbe-#K0M=$?@xq?&Wrtg+@vZAZ8w4jd_ya`9IMnoVXgqn#LqmXeXu>=7aHdEqeQ(h z3#GK%XH3jM0~$$0h$xHALSz}+JE(?5xXD_sjvMWo#irs#DtDa7CP?nLB^rdzxEz5OEUDIKRoXLAs=@j0K>y2SG&A zKPm0S=>T0&ntwPEDXz4(%6o8Z?R0jJ1EozlN(2ciZOXd~IPq&;HH$!L$|s178Y*qc z>eH`5X%4AG7_GG7-n@54(q#Pci%!NFM1TaPEw9IWytUwSPjgUOSr(Bbtq7K z1xhoxMZ{LGw1T%6QE9z*Ha`tYYnVp_N~N^*N1CG2p6&j6A6i7qA|g|?MI2p>Ey7}$ z>UL1t{9+=eW=ga9G36Vy&Ov^AzBBD6D_@{>4nN8J#Pet!|2$|i`vQr2DO=~ND}y3o zUS=I6LPY0f_!68K^VDO5fW>v$;IXllN;^|1z*4_1@3EbFy4~*pORPIlFU&%n*(CKz zodhh;yonG|mH|tV#qZ>mtH9DYY~fjkCH}bpi;gnyf^fx>*j~Ug@)%JsB}<>QF++f* zRx%MH%A)IpEZ?lhoClU%yIV?emFp5wEWn~V?j%a1!{xEffW^;&sF#wZBB=qsz!ScL z2oYuJ;DRjqakKC*@DzO5Hico?`bMIjmss;IOXe&{QU$6fzlbD;iaO(bH~lhNqTl*H zFF?Yxukr;?$vmWl;VT)Jk!5QaMBO*Y8o<}?xib+>ga@0W2ea(m@Y4uDEP-j&?U(oZlQABL@ zdM>+g4fUL5r&b5_{B|M{D3zYezPCU<>+R2)0qst829YV+B9vTl3a73OPy#)FoK3{k zOwTHJF8qYn8EbpahiNx~B?7H;!&Tm6I}3B_xj~cZx`e0~Zk^PO_br zOM?~2GSshKGOWw!Zsud+E0yN{S%78#B%adler{6=EPj26dSMpoajHqZ*W(S0Hzq`g zD2t36vh4FP#XpH*(fi_FhUN1&0hW55c|QSNV$JYr=WJu5UYLbCd(QlI;RUQqf;kZ) z%JO*?vJ~dt`2mfv(+k}j49lJ$67`(gj;H5lbH25Q6$<%4Br#N!o?k2R&Yo|K^1BOq zUiFO#sZP&HUwC>R*=|0*U#lXc$DCF;9zY|EnU6G6|%n-f8TdVaPt4fX7x|D+%2`HBXSQA0fkgjXB{J&jlw(;r`w>ZKf6I`AKggNm@Nq1I!2Un~#hlm6+8nRcK=Chu);a4j?-8W4)92!!D=_&?)C;#x>Vj}Ur+&rIIz9goA)@nA zyawlGP3w1%*&nNQ&u#`_ALPS~e z*CES9TT?|~@%wH1gki~9FHb*(zGXV^O2ADDp`!HMtj7(s#Kzg} zj)0zz|0F`H)AQ{S4RNL|t~lHRy0LOsd33^hPB_{Q>83}I#g~z~H6W7yNzcAbqBKCy z!Ky^0xSsQL_u|;@xF5~{J*%`Pf&}%PV8Z)dWHGwoV9;||2O^_}dVYRwniA;Qs~ZtU z>-o`2-m8nhX{l~(aCH-(6hEF5nH{U zmFM0^J&!Cq-vji#VI&bKm7e9NcSJpV^tOqB7SVPhktx<9-0OQ{i}=*D{ZG(y$LT~& z&Gc-x&JaJdu)R}yEYoh1eFR$P6l-4V9B=aZA~YFaJ)&N?by8=~{%6-uhj}UMPlSlg zi%dRmUjB@DwG}iVBlF-h!xG~sz_M;R?~F6C)$1d`GFg$R7iOU}VD{E|8nEiaez zLPS||HXzI9D?yE*d7SY#Nnuz@10*VK$h)1`ST}}8;3G(jf`}wxrFlHaMx`C;5t#}~ z(~KZOs#Ds6nY=U3Ig5`S0lLRWi8u)tgy*X9&N$6&Jn@Y4=Sd>zpOp5i=-3BP+K)3t zq`1-+IPjGAXJ2hsP?}vP5hSRz%%i+3@von}I}A#*&n7Zzs5DcZv137LCb>iyt+Zq6 z<8W8bG7X7^bxVCj1V~U?+gZG~EDn6$HyxBF|BOhIR%z4#TN#f04oZ`IOT<>Mw2I~h zsI=wU!OKBuGs}oTsgyQnNoQ2r*q)&+q20CrMP!P$h~$m97_C1fEd-?{)*8&Uh?*&F z@!bn}5q0RT9cfIv*|kTYbvFFQYn}bm2H8OCG|MCErEHx=-OEd0UOp5NA)@ngl{YUd zhH1714ZJkEVU$>@0k0qdmh-<7Q3HcMy^jZ$Q$vY*DOqfOUpWCRwZ{@6qAZ!4aE;#` z)WJV5KlIMc^$bgShyY85DetGow4<(gCE$AjQ7_Cw={aWlFZ?WteK8Rt%5rWqvZyb2 z{RQjNZGFg3hNU!2fJN4q_je_S$=cyxc~aR*)Jw_oG;%0@b9-n25hBWx?1d~zuJS#g z5$1M2r*pD$k9~-csAu`byysLKm1*PaVoJS;B!-GQ;|z1Tf_gSwWOM@bJY@h8Qk|ZA zZQ|*<_Rnt#K&Nd=#7WTe9(~^9R9&yFQiKb_nq!Ehf6}u@H$bUv~(4+IxCPnKaRI=4+jn@jUy`yGH%rCo7H(oba3`TPEyV<=VH zlmH|{aD@xQCbV$rt}oe-;+I(tdyj$ra$^-1znn_kj9NY0bT^zr@yi2GB>mX^g3(Je zyZGfDieD0gkPN};mo;6eaajTmne? zvHJz1dKPQ*WkB)Eb15W4aQbCgH>zJ;|9hhDuYK7txdrT(1Dmk;CHlxl)OeBlIU@}v zUShkE^ker6#(b^1>T)&|zqn07G6bhz7WAO{<)qeZCX|v?&Zv0?*e|V{S+p=q(z{t` rn~hIdGI*o7V}lKn0l3`Ja0tyE6W;LMK`8^ayC4}uS{d+OI9>q&r2g1< literal 0 HcmV?d00001 diff --git a/logs/self_tuning/ademamix_golden/study_0/librispeech_deepspeech_pytorch/trial_1/flags_0.json b/logs/self_tuning/ademamix_golden/study_0/librispeech_deepspeech_pytorch/trial_1/flags_0.json new file mode 100644 index 00000000..1e1f1367 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/librispeech_deepspeech_pytorch/trial_1/flags_0.json @@ -0,0 +1,62 @@ +{ + "logtostderr": false, + "alsologtostderr": false, + "log_dir": "", + "v": 0, + "verbosity": 0, + "logger_levels": {}, + "stderrthreshold": "fatal", + "showprefixforinfo": true, + "run_with_pdb": false, + "pdb_post_mortem": false, + "pdb": false, + "run_with_profiling": false, + "profile_file": null, + "use_cprofile_for_profiling": true, + "only_check_args": false, + "op_conversion_fallback_to_while_loop": true, + "delta_threshold": 0.5, + "tt_check_filter": false, + "tt_single_core_summaries": false, + "runtime_oom_exit": true, + "hbm_oom_exit": true, + "test_srcdir": "", + "test_tmpdir": "/tmp/absl_testing", + "test_random_seed": 301, + "test_randomize_ordering_seed": "", + "xml_output_file": "", + "experimental_orbax_use_distributed_process_id": false, + "experimental_orbax_use_distributed_barrier": false, + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "workload": "librispeech_deepspeech", + "tuning_ruleset": "self", + "tuning_search_space": null, + "num_tuning_trials": 1, + "data_dir": "/data/librispeech", + "imagenet_v2_data_dir": null, + "librispeech_tokenizer_vocab_path": "/data/librispeech/spm_model.vocab", + "framework": "pytorch", + "torch_compile": true, + "experiment_dir": "/experiment_runs", + "experiment_name": "submissions_a100/ademamix_golden/study_0", + "save_checkpoints": false, + "save_intermediate_checkpoints": true, + "resume_last_run": null, + "append_timestamp": false, + "use_wandb": false, + "profile": false, + "max_global_steps": null, + "overwrite": true, + "hparam_start_index": null, + "hparam_end_index": null, + "rng_seed": 1619867207, + "set_pytorch_max_split_size": false, + "pytorch_eval_num_workers": 0, + "capture_jax_trace": false, + "skip_evals": false, + "?": false, + "help": false, + "helpshort": false, + "helpfull": false, + "helpxml": false +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_0/librispeech_deepspeech_pytorch/trial_1/measurements.csv b/logs/self_tuning/ademamix_golden/study_0/librispeech_deepspeech_pytorch/trial_1/measurements.csv new file mode 100644 index 00000000..bc32aabd --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/librispeech_deepspeech_pytorch/trial_1/measurements.csv @@ -0,0 +1,286 @@ +global_step,grad_norm,loss,train/ctc_loss,train/wer,validation/ctc_loss,validation/wer,validation/num_examples,test/ctc_loss,test/wer,test/num_examples,score,total_duration,accumulated_submission_time,accumulated_eval_time,accumulated_logging_time,preemption_count +0,0.4999999701976776,33.75166702270508,,,,,,,,,,,,,, +1,,,31.939386414583453,2.4035197577988447,30.99260347598955,2.256911118621156,5348.0,31.05694238337957,2.3705238356386977,2472.0,34.61258101463318,107.63924741744997,34.61258101463318,72.46234583854675,0.0,0.0 +1,0.4999999701976776,33.18235778808594,,,,,,,,,,,,,, +2,0.4999999403953552,33.67015838623047,,,,,,,,,,,,,, +3,0.4999999701976776,33.65192413330078,,,,,,,,,,,,,, +4,0.4999999403953552,33.27874755859375,,,,,,,,,,,,,, +5,0.4999999403953552,33.46546173095703,,,,,,,,,,,,,, +6,0.5,33.547298431396484,,,,,,,,,,,,,, +7,0.4999999701976776,32.46466064453125,,,,,,,,,,,,,, +8,0.5,32.8299446105957,,,,,,,,,,,,,, +9,0.4999999403953552,32.46510696411133,,,,,,,,,,,,,, +10,0.5,32.14801025390625,,,,,,,,,,,,,, +11,0.5,32.20619201660156,,,,,,,,,,,,,, +12,0.5,32.202972412109375,,,,,,,,,,,,,, +13,0.5,31.426029205322266,,,,,,,,,,,,,, +14,0.4999999403953552,31.16885757446289,,,,,,,,,,,,,, +15,0.5,29.902210235595703,,,,,,,,,,,,,, +16,0.5,30.269081115722656,,,,,,,,,,,,,, +17,0.4999999701976776,29.510780334472656,,,,,,,,,,,,,, +18,0.5,29.19655418395996,,,,,,,,,,,,,, +19,0.4999999701976776,28.17563247680664,,,,,,,,,,,,,, +20,0.5,27.489368438720703,,,,,,,,,,,,,, +21,0.5000000596046448,27.066612243652344,,,,,,,,,,,,,, +22,0.5000000596046448,27.152585983276367,,,,,,,,,,,,,, +23,0.4999999701976776,26.04400634765625,,,,,,,,,,,,,, +24,0.5,25.90571403503418,,,,,,,,,,,,,, +25,0.5,25.45601463317871,,,,,,,,,,,,,, +26,0.5,24.512523651123047,,,,,,,,,,,,,, +27,0.4999999701976776,24.08442687988281,,,,,,,,,,,,,, +28,0.5,23.125377655029297,,,,,,,,,,,,,, +29,0.5,22.47698783874512,,,,,,,,,,,,,, +30,0.4999999701976776,21.746809005737305,,,,,,,,,,,,,, +31,0.4999999105930328,21.076322555541992,,,,,,,,,,,,,, +32,0.5,20.666139602661133,,,,,,,,,,,,,, +33,0.5,20.1121768951416,,,,,,,,,,,,,, +34,0.4999999403953552,19.392030715942383,,,,,,,,,,,,,, +35,0.4999998807907104,18.87059783935547,,,,,,,,,,,,,, +36,0.5,18.12567901611328,,,,,,,,,,,,,, +37,0.4999999701976776,16.817047119140625,,,,,,,,,,,,,, +38,0.4999999403953552,16.852699279785156,,,,,,,,,,,,,, +39,0.5,15.881023406982422,,,,,,,,,,,,,, +40,0.4999999701976776,15.67145538330078,,,,,,,,,,,,,, +41,0.4999999701976776,14.848554611206056,,,,,,,,,,,,,, +42,0.4999999105930328,14.422530174255373,,,,,,,,,,,,,, +43,0.5,13.9482421875,,,,,,,,,,,,,, +44,0.499999850988388,13.044654846191406,,,,,,,,,,,,,, +45,0.5,12.335125923156738,,,,,,,,,,,,,, +46,0.5,11.812021255493164,,,,,,,,,,,,,, +47,0.4999999403953552,11.57109546661377,,,,,,,,,,,,,, +48,0.4999999701976776,11.288980484008787,,,,,,,,,,,,,, +49,0.4999999403953552,10.918238639831545,,,,,,,,,,,,,, +50,0.4999999701976776,10.270544052124023,,,,,,,,,,,,,, +51,0.4999999701976776,9.947251319885254,,,,,,,,,,,,,, +52,0.4999999105930328,9.426873207092283,,,,,,,,,,,,,, +53,0.4999999105930328,9.328662872314451,,,,,,,,,,,,,, +54,0.4999999403953552,9.138580322265623,,,,,,,,,,,,,, +55,0.4999999403953552,8.828343391418457,,,,,,,,,,,,,, +56,0.4999999105930328,8.550689697265625,,,,,,,,,,,,,, +57,0.4999999105930328,8.490766525268555,,,,,,,,,,,,,, +58,0.4999998807907104,8.236915588378906,,,,,,,,,,,,,, +59,0.4999998807907104,8.030330657958984,,,,,,,,,,,,,, +60,0.4999998211860657,7.99373722076416,,,,,,,,,,,,,, +61,0.499999850988388,7.864288330078125,,,,,,,,,,,,,, +62,0.4999998211860657,7.630582809448242,,,,,,,,,,,,,, +63,0.499999850988388,7.525696754455566,,,,,,,,,,,,,, +64,0.4999998211860657,7.283774375915527,,,,,,,,,,,,,, +65,0.4999998211860657,7.2979736328125,,,,,,,,,,,,,, +66,0.4999997913837433,7.257999420166016,,,,,,,,,,,,,, +67,0.4999997615814209,7.079140186309815,,,,,,,,,,,,,, +68,0.4999997317790985,6.98946475982666,,,,,,,,,,,,,, +69,0.4999997615814209,7.093320369720459,,,,,,,,,,,,,, +70,0.4999997317790985,6.883864879608154,,,,,,,,,,,,,, +71,0.4999997317790985,6.857117652893066,,,,,,,,,,,,,, +72,0.4999996423721313,6.759705066680908,,,,,,,,,,,,,, +73,0.4999996721744537,6.789670944213867,,,,,,,,,,,,,, +74,0.4999996721744537,6.71970796585083,,,,,,,,,,,,,, +75,0.4999997317790985,6.75324010848999,,,,,,,,,,,,,, +76,0.4999998211860657,6.584537506103516,,,,,,,,,,,,,, +77,0.4999997019767761,6.614601135253906,,,,,,,,,,,,,, +78,0.4999996125698089,6.578903675079346,,,,,,,,,,,,,, +79,0.4999997019767761,6.625402927398682,,,,,,,,,,,,,, +80,0.4999997615814209,6.582681179046631,,,,,,,,,,,,,, +81,0.4999997019767761,6.497599124908447,,,,,,,,,,,,,, +82,0.4999997019767761,6.490114212036133,,,,,,,,,,,,,, +83,0.4999996423721313,6.45712947845459,,,,,,,,,,,,,, +84,0.4999997317790985,6.482800006866455,,,,,,,,,,,,,, +85,0.4999998211860657,6.431879997253418,,,,,,,,,,,,,, +86,0.4999996721744537,6.3827314376831055,,,,,,,,,,,,,, +87,0.4999996125698089,6.3644328117370605,,,,,,,,,,,,,, +88,0.4999997019767761,6.357175350189209,,,,,,,,,,,,,, +89,0.4999996423721313,6.324764728546143,,,,,,,,,,,,,, +90,0.4999997019767761,6.333850860595703,,,,,,,,,,,,,, +91,0.4999998807907104,6.299194812774658,,,,,,,,,,,,,, +92,0.4999998211860657,6.271178245544434,,,,,,,,,,,,,, +93,0.4999997615814209,6.249658584594727,,,,,,,,,,,,,, +94,0.4999997019767761,6.216371059417725,,,,,,,,,,,,,, +95,0.4999995529651642,6.234154224395752,,,,,,,,,,,,,, +96,0.4999997615814209,6.204021453857422,,,,,,,,,,,,,, +97,0.4999996125698089,6.181948661804199,,,,,,,,,,,,,, +98,0.4999995529651642,6.160825729370117,,,,,,,,,,,,,, +99,0.4999994039535522,6.154407024383545,,,,,,,,,,,,,, +100,0.4999997019767761,6.146690368652344,,,,,,,,,,,,,, +500,0.4999997913837433,4.332629203796387,,,,,,,,,,,,,, +1000,0.4999997317790985,3.004477024078369,,,,,,,,,,,,,, +1106,,,6.406972739363964,0.9413664109251888,6.318610137884268,0.8966832424081495,5348.0,6.257749290276792,0.8995389271423638,2472.0,1480.9474048614502,1597.7229297161102,1480.9474048614502,114.427020072937,0.0361976623535156,0.0 +1500,0.4999997615814209,2.620929956436157,,,,,,,,,,,,,, +2000,0.4999998807907104,2.3482532501220703,,,,,,,,,,,,,, +2500,0.4999998211860657,2.138546705245972,,,,,,,,,,,,,, +2883,,,4.940423060307439,0.9072633120790204,4.846255274261603,0.8700429681842321,5348.0,4.628748628943803,0.864663944914996,2472.0,2926.629233121872,3090.896448850632,2926.629233121872,159.63744711875916,0.0753614902496337,0.0 +3000,0.499999850988388,2.061686992645264,,,,,,,,,,,,,, +3500,0.499999850988388,1.9733930826187127,,,,,,,,,,,,,, +4000,0.499999850988388,1.836808562278748,,,,,,,,,,,,,, +4500,0.499999850988388,1.8278149366378784,,,,,,,,,,,,,, +4783,,,0.6839214356055066,0.2145233570561233,0.9085786806811332,0.2542075025346401,5348.0,0.5881865362281438,0.1848150630674547,2472.0,4372.030340671539,4585.448388338089,4372.030340671539,206.4729919433593,0.134275197982788,0.0 +5000,0.4999998211860657,1.7696833610534668,,,,,,,,,,,,,, +5500,0.4999997615814209,1.8473175764083865,,,,,,,,,,,,,, +6000,0.4999998807907104,1.7039304971694946,,,,,,,,,,,,,, +6500,0.4999997615814209,1.6685926914215088,,,,,,,,,,,,,, +6654,,,0.498515827768839,0.1641555358319402,0.736931927883891,0.213334620769565,5348.0,0.4538967231756887,0.146385554404566,2472.0,5817.834228515625,6079.034798622131,5817.834228515625,251.97098755836487,0.1728487014770507,0.0 +7000,0.4999997615814209,1.5716161727905271,,,,,,,,,,,,,, +7500,0.4999997317790985,1.5868427753448486,,,,,,,,,,,,,, +8000,0.4999997913837433,1.541409254074097,,,,,,,,,,,,,, +8446,,,0.4237485521826537,0.1401586722497808,0.672041145126959,0.1918794959687153,5348.0,0.3912555346312665,0.1276176548250157,2472.0,7263.678015470505,7572.289734840393,7263.678015470505,297.12858510017395,0.2118051052093505,0.0 +8500,0.4999997317790985,1.5005810260772705,,,,,,,,,,,,,, +9000,0.4999998211860657,1.5772579908370972,,,,,,,,,,,,,, +9500,0.4999998211860657,1.5967402458190918,,,,,,,,,,,,,, +10000,0.499999850988388,1.5338062047958374,,,,,,,,,,,,,, +10260,,,0.37034147229205,0.124100866326525,0.6151254756253768,0.1764012938734128,5348.0,0.350934586505581,0.1148010480775089,2472.0,8711.645572900772,9067.41434621811,8711.645572900772,342.0227270126343,0.2510788440704345,0.0 +10500,0.4999998807907104,1.5209287405014038,,,,,,,,,,,,,, +11000,0.4999998807907104,1.5171277523040771,,,,,,,,,,,,,, +11500,0.4999998211860657,1.421940803527832,,,,,,,,,,,,,, +12000,0.4999998211860657,1.5268211364746094,,,,,,,,,,,,,, +12195,,,0.3353088900237093,0.1132540879612739,0.5805158808644766,0.167498672331386,5348.0,0.328308681205239,0.1061483151544695,2472.0,10159.301994562147,10564.632512331007,10159.301994562147,388.9419791698456,0.2914483547210693,0.0 +12500,0.4999998211860657,1.3726520538330078,,,,,,,,,,,,,, +13000,0.4999998807907104,1.4768458604812622,,,,,,,,,,,,,, +13500,0.4999998807907104,1.321739673614502,,,,,,,,,,,,,, +14000,0.4999998807907104,1.3973541259765625,,,,,,,,,,,,,, +14124,,,0.314595242569997,0.1074658723978894,0.5686893106728451,0.1635784290059383,5348.0,0.3159095949738693,0.1031218897893689,2472.0,11605.590642929075,12058.94044137001,11605.590642929075,434.5560202598572,0.4015920162200928,0.0 +14500,0.499999850988388,1.3541994094848633,,,,,,,,,,,,,, +15000,0.4999998211860657,1.355735421180725,,,,,,,,,,,,,, +15500,0.499999850988388,1.3410564661026,,,,,,,,,,,,,, +16000,0.499999850988388,1.3418275117874146,,,,,,,,,,,,,, +16026,,,0.2964337983923912,0.1024290902754711,0.5432022950101718,0.1564235021484092,5348.0,0.3044302253371185,0.0988361464871123,2472.0,13051.91927099228,13555.112949848177,13051.91927099228,481.9424600601196,0.461036205291748,0.0 +16500,0.4999998807907104,1.3864673376083374,,,,,,,,,,,,,, +17000,0.4999999105930328,1.2627692222595217,,,,,,,,,,,,,, +17500,0.499999850988388,1.360073208808899,,,,,,,,,,,,,, +17854,,,0.2800245639965031,0.0967770039586384,0.536387646847373,0.1541350842466084,5348.0,0.2936940740128395,0.094266041070014,2472.0,14497.74052286148,15048.5044631958,14497.74052286148,527.1802635192871,0.5412375926971436,0.0 +18000,0.4999998807907104,1.4030377864837646,,,,,,,,,,,,,, +18500,0.4999999105930328,1.3778812885284424,,,,,,,,,,,,,, +19000,0.4999999105930328,1.357402801513672,,,,,,,,,,,,,, +19500,0.4999999403953552,1.3000246286392212,,,,,,,,,,,,,, +19618,,,0.2723956721104032,0.094386574389188,0.5301783712891802,0.1519915029208709,5348.0,0.2908651364604168,0.0940426136940669,2472.0,15943.188425302504,16541.64181613922,15943.188425302504,572.5648229122162,0.5980556011199951,0.0 +20000,0.4999999105930328,1.3385708332061768,,,,,,,,,,,,,, +20500,0.4999999403953552,1.3948860168457031,,,,,,,,,,,,,, +21000,0.4999998807907104,1.379390835762024,,,,,,,,,,,,,, +21500,0.499999850988388,1.3555381298065186,,,,,,,,,,,,,, +21517,,,0.2613459662660684,0.0916367636088407,0.5153653583829365,0.1470380920195046,5348.0,0.2821661881411704,0.0907115146344931,2472.0,17389.614331007004,18037.02626419068,17389.614331007004,618.8605799674988,0.6618938446044922,0.0 +22000,0.4999998807907104,1.360029697418213,,,,,,,,,,,,,, +22500,0.4999999105930328,1.3734184503555298,,,,,,,,,,,,,, +23000,0.499999850988388,1.3448195457458496,,,,,,,,,,,,,, +23454,,,0.2536218819839647,0.0889359593572521,0.515249051888688,0.1471539612803553,5348.0,0.2779084285679721,0.0894725082769687,2472.0,18836.100885391235,19532.2314286232,18836.100885391235,665.1551597118378,0.762861967086792,0.0 +23500,0.4999999701976776,1.3234541416168213,,,,,,,,,,,,,, +24000,0.4999999105930328,1.3212579488754272,,,,,,,,,,,,,, +24500,0.499999850988388,1.3062100410461426,,,,,,,,,,,,,, +25000,0.4999999403953552,1.3487989902496338,,,,,,,,,,,,,, +25378,,,0.2348187262523939,0.0827012398651775,0.4875852750778581,0.1403756095205909,5348.0,0.2664396442915672,0.0866288871285519,2472.0,20281.97596359253,21026.32065486908,20281.97596359253,710.9896395206451,0.8335576057434082,0.0 +25500,0.499999850988388,1.3311156034469604,,,,,,,,,,,,,, +26000,0.4999999701976776,1.3619003295898438,,,,,,,,,,,,,, +26500,0.4999999105930328,1.303876519203186,,,,,,,,,,,,,, +27000,0.4999999403953552,1.338093876838684,,,,,,,,,,,,,, +27251,,,0.2219891192593298,0.0785247945809669,0.4785246999478852,0.1367353835755322,5348.0,0.2573770888444416,0.0821806511892429,2472.0,21727.520691156387,22520.22661948204,21727.520691156387,757.0047874450684,0.9037110805511476,0.0 +27500,0.5,1.2549574375152588,,,,,,,,,,,,,, +28000,0.5,1.2664973735809326,,,,,,,,,,,,,, +28500,0.4999999403953552,1.1696932315826416,,,,,,,,,,,,,, +29000,0.4999998807907104,1.1944557428359983,,,,,,,,,,,,,, +29040,,,0.2104059692594999,0.073662257894135,0.4613453330633413,0.1321102689132429,5348.0,0.2483214390847796,0.0787479942315113,2472.0,23173.42985391617,24013.617489099503,23173.42985391617,802.1866991519928,0.9844353199005128,0.0 +29500,0.4999999105930328,1.2367489337921145,,,,,,,,,,,,,, +30000,0.4999999403953552,1.207222580909729,,,,,,,,,,,,,, +30500,0.4999999105930328,1.169906497001648,,,,,,,,,,,,,, +30860,,,0.1980789526051357,0.0695511546482692,0.454876955479581,0.1291556027615507,5348.0,0.2418787603232466,0.0772043141795137,2472.0,24620.16385602951,25508.309183359142,24620.16385602951,847.7015099525452,1.0710821151733398,0.0 +31000,0.4999999403953552,1.212024807929993,,,,,,,,,,,,,, +31500,0.4999999403953552,1.1333380937576294,,,,,,,,,,,,,, +32000,0.4999998807907104,1.2006267309188845,,,,,,,,,,,,,, +32500,0.4999999105930328,1.1943844556808472,,,,,,,,,,,,,, +32795,,,0.1867580999108773,0.0659845683886108,0.4414691859899287,0.1253029498382658,5348.0,0.2354236815681656,0.074360693031097,2472.0,26066.00140094757,27001.54048967361,26066.00140094757,892.7218158245087,1.1452579498291016,0.0 +33000,0.4999999403953552,1.2133270502090454,,,,,,,,,,,,,, +33500,0.4999999701976776,1.186384677886963,,,,,,,,,,,,,, +34000,0.4999998807907104,1.1334519386291504,,,,,,,,,,,,,, +34500,0.4999999701976776,1.1280875205993652,,,,,,,,,,,,,, +34725,,,0.1802414469286712,0.0634688999123327,0.4364551055951125,0.1239607975667455,5348.0,0.2321519231079424,0.0738732151199398,2472.0,27511.845928907394,28497.04074501992,27511.845928907394,940.0010640621184,1.217942714691162,0.0 +35000,0.499999850988388,1.131145477294922,,,,,,,,,,,,,, +35500,0.4999999403953552,1.156831979751587,,,,,,,,,,,,,, +36000,0.4999999403953552,1.1825071573257446,,,,,,,,,,,,,, +36500,0.4999999403953552,1.188075065612793,,,,,,,,,,,,,, +36627,,,0.1760754508430931,0.0619769233701245,0.4329456025059021,0.1222710375126732,5348.0,0.2286605688270211,0.0720045497938374,2472.0,28958.37071299553,29993.15261864662,28958.37071299553,986.9447524547576,1.3041667938232422,0.0 +37000,0.4999999701976776,1.1605700254440308,,,,,,,,,,,,,, +37500,0.4999999105930328,1.1737715005874634,,,,,,,,,,,,,, +38000,0.4999999701976776,1.1675797700881958,,,,,,,,,,,,,, +38456,,,0.1753784761435418,0.061573980800331,0.4328299337106941,0.1221262009366098,5348.0,0.2286049956852055,0.0719233034753112,2472.0,30403.793350219727,31486.40464568138,30403.793350219727,1032.3531494140625,1.3909907341003418,0.0 +38500,0.4999999105930328,1.1380921602249146,,,,,,,,,,,,,, +39000,0.4999999403953552,1.1359905004501345,,,,,,,,,,,,,, +39500,0.4999999105930328,1.1752535104751587,,,,,,,,,,,,,, +40000,0.4999998807907104,1.1258069276809692,,,,,,,,,,,,,, +40228,,,0.1746056823748116,0.0610839155127444,0.432282542665009,0.1220972336213972,5348.0,0.2285071567117233,0.0721467308512583,2472.0,31849.34828710556,32979.711805820465,31849.34828710556,1077.666915655136,1.4649040699005127,0.0 +40500,0.499999850988388,1.172392725944519,,,,,,,,,,,,,, +41000,0.499999850988388,1.141939997673035,,,,,,,,,,,,,, +41500,0.4999999403953552,1.1755826473236084,,,,,,,,,,,,,, +42000,0.4999999701976776,1.1628968715667725,,,,,,,,,,,,,, +42123,,,0.1733037894971375,0.0612200447592962,0.4306325699781494,0.1218365277844831,5348.0,0.2278372798245048,0.0724920277049946,2472.0,33295.38259077072,34474.33810329437,33295.38259077072,1123.758908510208,1.5458810329437256,0.0 +42500,0.4999999403953552,1.1815471649169922,,,,,,,,,,,,,, +43000,0.5,1.138005614280701,,,,,,,,,,,,,, +43500,0.4999999701976776,1.126315951347351,,,,,,,,,,,,,, +44000,0.4999998807907104,1.1484259366989136,,,,,,,,,,,,,, +44057,,,0.1736082882334332,0.0619823685399866,0.433137451809574,0.1230917781103654,5348.0,0.2299340884895799,0.0735685414254666,2472.0,34742.66511321068,35969.98772978783,34742.66511321068,1169.7595500946045,1.6316978931427002,0.0 +44500,0.499999850988388,1.1936637163162231,,,,,,,,,,,,,, +45000,0.5,1.2018557786941528,,,,,,,,,,,,,, +45500,0.4999999105930328,1.2303271293640137,,,,,,,,,,,,,, +45979,,,0.176779414822418,0.0622818528824006,0.4403990288922795,0.1245691111862115,5348.0,0.2336581946819149,0.07413726565515,2472.0,36188.63485765457,37465.13506793976,36188.63485765457,1216.3707485198977,1.7183234691619873,0.0 +46000,0.4999999105930328,1.11965012550354,,,,,,,,,,,,,, +46500,0.4999999105930328,1.211846113204956,,,,,,,,,,,,,, +47000,0.4999998807907104,1.1769431829452517,,,,,,,,,,,,,, +47500,0.4999999403953552,1.1763243675231934,,,,,,,,,,,,,, +47856,,,0.1866825997441976,0.0667033308104046,0.4603885392335995,0.1301211799353063,5348.0,0.2408211820117426,0.0766152783701988,2472.0,37634.08888673782,38958.67030453682,37634.08888673782,1261.9813685417175,1.7793095111846924,0.0 +48000,0.4999999403953552,1.2061699628829956,,,,,,,,,,,,,, +48500,0.4999999701976776,1.2581828832626345,,,,,,,,,,,,,, +49000,0.4999999105930328,1.2749943733215332,,,,,,,,,,,,,, +49500,0.4999999701976776,1.181243896484375,,,,,,,,,,,,,, +49651,,,0.1944978369906489,0.0705367303933046,0.472085950842626,0.1332979288369623,5348.0,0.2506045752951803,0.0811244490484024,2472.0,39079.88468909264,40452.2503619194,39079.88468909264,1307.3615272045135,1.8513941764831543,0.0 +50000,0.4999999403953552,1.2770346403121948,,,,,,,,,,,,,, +50500,0.499999850988388,1.101345419883728,,,,,,,,,,,,,, +51000,0.4999999403953552,1.171238899230957,,,,,,,,,,,,,, +51465,,,0.2047566513494253,0.0740216391050318,0.4808824695160237,0.1350263119779848,5348.0,0.2555358410220014,0.0831556070115572,2472.0,40525.51788115501,41946.51808142662,40525.51788115501,1353.447606086731,1.956232547760009,0.0 +51500,0.4999999105930328,1.2293148040771484,,,,,,,,,,,,,, +52000,0.4999999403953552,1.2078243494033811,,,,,,,,,,,,,, +52500,0.4999999105930328,1.1939834356307983,,,,,,,,,,,,,, +53000,0.4999999105930328,1.2989579439163208,,,,,,,,,,,,,, +53400,,,0.2253668548049677,0.0806484108271757,0.4989853227188818,0.1406942499879303,5348.0,0.2687973568859281,0.0860398513192371,2472.0,41970.87526202202,43439.98548531532,41970.87526202202,1399.1608033180237,2.0289409160614014,0.0 +53500,0.4999998807907104,1.281888723373413,,,,,,,,,,,,,, +54000,0.4999999403953552,1.276806116104126,,,,,,,,,,,,,, +54500,0.499999850988388,1.3454179763793943,,,,,,,,,,,,,, +55000,0.5,1.3220462799072266,,,,,,,,,,,,,, +55331,,,0.240866233795501,0.0856035154016629,0.5176834925030138,0.1459180224979481,5348.0,0.2769799916526873,0.0877663355879186,2472.0,43416.44570946693,44934.14357638359,43416.44570946693,1445.3778800964355,2.102470874786377,0.0 +55500,0.4999999105930328,1.28280770778656,,,,,,,,,,,,,, +56000,0.4999999105930328,1.337410569190979,,,,,,,,,,,,,, +56500,0.4999999701976776,1.3311973810195925,,,,,,,,,,,,,, +57000,0.4999999105930328,1.2640831470489502,,,,,,,,,,,,,, +57234,,,0.2501228837630154,0.089447805324287,0.5302517557326201,0.1483512769758123,5348.0,0.2789501036357184,0.0898584282899681,2472.0,44862.44470882416,46430.06482982636,44862.44470882416,1492.5967779159546,2.17545223236084,0.0 +57500,0.4999999105930328,1.3188778162002563,,,,,,,,,,,,,, +58000,0.4999998807907104,1.3274576663970947,,,,,,,,,,,,,, +58500,0.4999999701976776,1.322802186012268,,,,,,,,,,,,,, +59000,0.4999998807907104,1.2478609085083008,,,,,,,,,,,,,, +59070,,,0.2561507256520068,0.0905967361651846,0.5335001455915963,0.1492685752908801,5348.0,0.2836022434915156,0.0919505209920175,2472.0,46307.86073827744,47923.331095695496,46307.86073827744,1538.037742137909,2.2553555965423584,0.0 +59500,0.4999999403953552,1.2923192977905271,,,,,,,,,,,,,, +60000,0.4999999701976776,1.3277688026428225,,,,,,,,,,,,,, +60500,0.5,1.3613649606704712,,,,,,,,,,,,,, +60843,,,0.2732018576283178,0.0970492624517421,0.5415587974212628,0.1533239994206537,5348.0,0.295044841602684,0.0956675400645908,2472.0,47753.28298354149,49416.46483445168,47753.28298354149,1583.370682001114,2.3385322093963623,0.0 +61000,0.5,1.3961572647094729,,,,,,,,,,,,,, +61500,0.4999998807907104,1.29708993434906,,,,,,,,,,,,,, +62000,0.4999998807907104,1.3286187648773191,,,,,,,,,,,,,, +62500,0.4999998807907104,1.416536808013916,,,,,,,,,,,,,, +62733,,,0.2780718176932209,0.0973433016242941,0.5457853195009544,0.153208130159803,5348.0,0.2982510716417188,0.0948347652996973,2472.0,49198.828130960464,50911.28756451607,49198.828130960464,1630.123468875885,2.4058196544647217,0.0 +63000,0.4999999105930328,1.3499302864074707,,,,,,,,,,,,,, +63500,0.4999998807907104,1.3608038425445557,,,,,,,,,,,,,, +64000,0.499999850988388,1.4167057275772097,,,,,,,,,,,,,, +64500,0.5,1.3918771743774414,,,,,,,,,,,,,, +64663,,,0.2776010644540219,0.0967062167504315,0.546718027928471,0.154666151692174,5348.0,0.2912591638976708,0.0934942010440152,2472.0,50644.459433078766,52405.86308503151,50644.459433078766,1676.6409544944763,2.5173280239105225,0.0 +65000,0.4999999403953552,1.342724084854126,,,,,,,,,,,,,, +65500,0.4999999701976776,1.3381001949310305,,,,,,,,,,,,,, +66000,0.4999998807907104,1.299668550491333,,,,,,,,,,,,,, +66500,0.4999999701976776,1.3684017658233645,,,,,,,,,,,,,, +66583,,,0.2903884168659453,0.1006757455798833,0.5558291774977396,0.1578815236807801,5348.0,0.2992008052938899,0.0968049885239575,2472.0,52089.85448694229,53900.25670194626,52089.85448694229,1723.1894779205322,2.5836801528930664,0.0 +67000,0.4999999105930328,1.3700971603393557,,,,,,,,,,,,,, +67500,0.4999999701976776,1.417709231376648,,,,,,,,,,,,,, +68000,0.4999999403953552,1.3768389225006104,,,,,,,,,,,,,, +68464,,,0.2889195519547448,0.1000223251964345,0.5444093611864577,0.1543861343117848,5348.0,0.2996781042325311,0.0974955822314301,2472.0,53535.67465877533,55394.11697602272,53535.67465877533,1768.7811193466189,2.64736270904541,0.0 +68500,0.4999999403953552,1.3825395107269287,,,,,,,,,,,,,, +69000,0.4999999403953552,1.4139329195022583,,,,,,,,,,,,,, +69500,0.4999999701976776,1.3692858219146729,,,,,,,,,,,,,, +70000,0.4999999701976776,1.4652100801467896,,,,,,,,,,,,,, +70255,,,0.2840348426492549,0.098366993558364,0.5450164997268686,0.1535171148554048,5348.0,0.2999013797261113,0.0964800032498527,2472.0,54981.33536720276,56888.05970978737,54981.33536720276,1814.6919968128204,2.7203445434570312,0.0 +70500,0.4999999701976776,1.3952243328094482,,,,,,,,,,,,,, +71000,0.4999999403953552,1.4014217853546145,,,,,,,,,,,,,, +71500,0.4999999701976776,1.3738051652908323,,,,,,,,,,,,,, +72000,0.5,1.3888047933578491,,,,,,,,,,,,,, +72069,,,,,,,,,,,56426.86305117607,,,,,0.0 diff --git a/logs/self_tuning/ademamix_golden/study_0/librispeech_deepspeech_pytorch/trial_1/meta_data_0.json b/logs/self_tuning/ademamix_golden/study_0/librispeech_deepspeech_pytorch/trial_1/meta_data_0.json new file mode 100644 index 00000000..c182d743 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/librispeech_deepspeech_pytorch/trial_1/meta_data_0.json @@ -0,0 +1,77 @@ +{ + "workload.attention_temperature": 1.0, + "workload.enable_decoder_layer_norm": true, + "workload.enable_residual_connections": true, + "workload.eval_batch_size": 256, + "workload.eval_num_workers": 0, + "workload.eval_period_time_sec": 1447, + "workload.freq_mask_count": 2, + "workload.layernorm_everywhere": false, + "workload.max_allowed_runtime_sec": 36949, + "workload.num_eval_train_examples": 5376, + "workload.num_test_examples": 2472, + "workload.num_train_examples": 263840, + "workload.num_validation_examples": 5348, + "workload.requires_sync_before_eval": false, + "workload.step_hint": 38400, + "workload.target_metric_name": "wer", + "workload.test_target_value": 0.074143, + "workload.time_mask_count": 10, + "workload.use_gelu": false, + "workload.use_post_layer_norm": true, + "workload.use_specaug": true, + "workload.use_tanh": false, + "workload.validation_target_value": 0.119936, + "cpu.util.avg_percent_since_last": 15.2, + "cpu.freq.current": 2200.2180000000003, + "mem.total": 359053524992, + "mem.available": 348482576384, + "mem.used": 7415132160, + "mem.percent_used": 2.9, + "mem.read_bytes_since_boot": 1055256064, + "mem.write_bytes_since_boot": 30541989376, + "net.bytes_sent_since_boot": 31247566, + "net.bytes_recv_since_boot": 31250985, + "gpu.count": 4, + "gpu.0.compute.util": 0.0, + "gpu.0.mem.util": 0.0309326171875, + "gpu.0.mem.total": 40960.0, + "gpu.0.mem.used": 1267.0, + "gpu.0.mem.free": 39060.0, + "gpu.0.temp.current": 35.0, + "gpu.1.compute.util": 0.0, + "gpu.1.mem.util": 0.0309326171875, + "gpu.1.mem.total": 40960.0, + "gpu.1.mem.used": 1267.0, + "gpu.1.mem.free": 39060.0, + "gpu.1.temp.current": 34.0, + "gpu.2.compute.util": 0.0, + "gpu.2.mem.util": 0.0309326171875, + "gpu.2.mem.total": 40960.0, + "gpu.2.mem.used": 1267.0, + "gpu.2.mem.free": 39060.0, + "gpu.2.temp.current": 33.0, + "gpu.3.compute.util": 0.0, + "gpu.3.mem.util": 0.0309326171875, + "gpu.3.mem.total": 40960.0, + "gpu.3.mem.used": 1267.0, + "gpu.3.mem.free": 39060.0, + "gpu.3.temp.current": 34.0, + "gpu.avg.compute.util": 0.0, + "gpu.avg.mem.util": 0.0309326171875, + "gpu.avg.mem.total": 40960.0, + "gpu.avg.mem.used": 1267.0, + "gpu.avg.mem.free": 39060.0, + "gpu.avg.temp.current": 34.0, + "os_platform": "Linux-6.1.0-44-cloud-amd64-x86_64-with-glibc2.31", + "python_version": "3.11.10", + "python_compiler": "GCC 9.4.0", + "git_branch": "main", + "git_commit_hash": "b21be29be0a1573fb4f78f849aea019cdb520862", + "cpu_model_name": "Intel(R) Xeon(R) CPU @ 2.20GHz", + "cpu_count": 24, + "gpu_model_name": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_driver": "550.90.12", + "rng_seed": 1619867207 +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/ogbg_pytorch_09-16-2026-07-32-45.log b/logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/ogbg_pytorch_09-16-2026-07-32-45.log new file mode 100644 index 00000000..89437064 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/ogbg_pytorch_09-16-2026-07-32-45.log @@ -0,0 +1,452 @@ +torchrun --redirects 1:0,2:0,3:0 --standalone --nnodes=1 --nproc_per_node=4 submission_runner.py --framework=pytorch --workload=ogbg --submission_path=submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py --data_dir=/data/ogbg --experiment_dir=/experiment_runs --experiment_name=submissions_a100/ademamix_golden/study_0 --overwrite=True --save_checkpoints=False --rng_seed=1297901242 --torch_compile=true --tuning_ruleset=self 2>&1 | tee -a /logs/ogbg_pytorch_09-16-2026-07-32-45.log +W0916 07:32:51.291000 9 site-packages/torch/distributed/run.py:803] +W0916 07:32:51.291000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0916 07:32:51.291000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0916 07:32:51.291000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-16 07:32:54.375693: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 07:32:54.375673: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 07:32:54.375673: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 07:32:54.375673: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789543974.400377 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789543974.400376 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789543974.400377 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789543974.400376 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789543974.408472 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789543974.408491 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789543974.408495 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789543974.408509 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789543974.434986 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789543974.434986 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789543974.434986 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789543974.434986 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789543974.435017 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789543974.435017 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789543974.435017 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789543974.435019 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789543974.435019 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789543974.435020 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789543974.435021 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789543974.435022 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789543974.435022 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789543974.435023 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789543974.435025 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789543974.435027 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789543980.216056 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789543980.283409 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789543980.386929 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789543980.478617 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W916 07:33:04.184250247 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0916 07:33:05.634359 140272984544448 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/ogbg_pytorch. +I0916 07:33:05.634354 140526571857088 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/ogbg_pytorch. +I0916 07:33:05.634377 140425199580352 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/ogbg_pytorch. +I0916 07:33:05.634356 139777939743936 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/ogbg_pytorch. +I0916 07:33:05.660536 140425199580352 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/ogbg_pytorch/trial_1. +I0916 07:33:05.944851 140425199580352 submission_runner.py:242] Initializing dataset. +I0916 07:33:05.945038 140425199580352 submission_runner.py:251] Initializing model. +W0916 07:33:06.929901 140526571857088 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0916 07:33:06.929987 140425199580352 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0916 07:33:06.930208 139777939743936 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0916 07:33:06.930386 140272984544448 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +I0916 07:33:08.364084 140425199580352 submission_runner.py:294] Initializing optimizer. +I0916 07:33:08.364726 140425199580352 submission_runner.py:299] Initializing metrics bundle. +I0916 07:33:08.364871 140425199580352 submission_runner.py:321] Initializing checkpoint and logger. +I0916 07:33:08.366340 140425199580352 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_0/ogbg_pytorch/trial_1/meta_data_0.json. +I0916 07:33:08.366534 140425199580352 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 07:33:08.366593 140425199580352 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 07:33:08.367903 139777939743936 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 07:33:08.368059 139777939743936 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 07:33:08.368367 140272984544448 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 07:33:08.368541 140272984544448 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 07:33:08.370305 140526571857088 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 07:33:08.370463 140526571857088 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 07:33:08.572136 140425199580352 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_0/ogbg_pytorch/trial_1/flags_0.json. +I0916 07:33:08.619366 140425199580352 submission_runner.py:359] Starting training loop. +I0916 07:33:10.633972 140425199580352 dataset_info.py:707] Load dataset info from /data/ogbg/ogbg_molpcba/0.1.3 +I0916 07:33:10.671423 140425199580352 reader.py:262] Creating a tf.data.Dataset reading 8 files located in folders: /data/ogbg/ogbg_molpcba/0.1.3. +WARNING:tensorflow:From /usr/local/lib/python3.11/site-packages/tensorflow_datasets/core/reader.py:102: CounterV2 (from tensorflow.python.data.experimental.ops.counter) is deprecated and will be removed in a future version. +Instructions for updating: +Use `tf.data.Dataset.counter(...)` instead. +W0916 07:33:10.791254 140425199580352 deprecation.py:50] From /usr/local/lib/python3.11/site-packages/tensorflow_datasets/core/reader.py:102: CounterV2 (from tensorflow.python.data.experimental.ops.counter) is deprecated and will be removed in a future version. +Instructions for updating: +Use `tf.data.Dataset.counter(...)` instead. +I0916 07:33:11.240701 140425199580352 logging_logger.py:49] Constructing tf.data.Dataset ogbg_molpcba for split train, from /data/ogbg/ogbg_molpcba/0.1.3 +I0916 07:33:19.909852 140401247602432 logging_writer.py:48] [0] global_step=0, grad_norm=0.5, loss=0.756861 +I0916 07:33:19.948625 140425199580352 submission.py:307] 0) loss = 0.757, grad_norm = 0.500 +I0916 07:33:20.371079 140425199580352 spec.py:333] Evaluating on the training split. +I0916 07:33:20.374310 140425199580352 dataset_info.py:707] Load dataset info from /data/ogbg/ogbg_molpcba/0.1.3 +I0916 07:33:20.377544 140425199580352 reader.py:262] Creating a tf.data.Dataset reading 8 files located in folders: /data/ogbg/ogbg_molpcba/0.1.3. +I0916 07:33:20.439122 140425199580352 logging_logger.py:49] Constructing tf.data.Dataset ogbg_molpcba for split train, from /data/ogbg/ogbg_molpcba/0.1.3 +I0916 07:33:58.567810 140425199580352 spec.py:346] Evaluating on the validation split. +I0916 07:33:58.570092 140425199580352 dataset_info.py:707] Load dataset info from /data/ogbg/ogbg_molpcba/0.1.3 +I0916 07:33:58.573212 140425199580352 reader.py:262] Creating a tf.data.Dataset reading 1 files located in folders: /data/ogbg/ogbg_molpcba/0.1.3. +I0916 07:33:58.633522 140425199580352 logging_logger.py:49] Constructing tf.data.Dataset ogbg_molpcba for split validation, from /data/ogbg/ogbg_molpcba/0.1.3 +I0916 07:34:17.239961 140425199580352 spec.py:363] Evaluating on the test split. +I0916 07:34:17.242381 140425199580352 dataset_info.py:707] Load dataset info from /data/ogbg/ogbg_molpcba/0.1.3 +I0916 07:34:17.245832 140425199580352 reader.py:262] Creating a tf.data.Dataset reading 1 files located in folders: /data/ogbg/ogbg_molpcba/0.1.3. +I0916 07:34:17.309720 140425199580352 logging_logger.py:49] Constructing tf.data.Dataset ogbg_molpcba for split test, from /data/ogbg/ogbg_molpcba/0.1.3 +I0916 07:34:35.930121 140425199580352 submission_runner.py:516] Time since start: 87.31s, Step: 1, {'train/accuracy': 0.5281778531601953, 'train/loss': 0.7542170812830864, 'train/mean_average_precision': 0.023033742192316613, 'validation/accuracy': 0.5284877849184416, 'validation/loss': 0.7498691400731963, 'validation/mean_average_precision': 0.027189786787325653, 'validation/num_examples': 43793, 'test/accuracy': 0.5292198959097536, 'test/loss': 0.7482351910609405, 'test/mean_average_precision': 0.028311290438915268, 'test/num_examples': 43793, 'score': 11.330527305603027, 'total_duration': 87.31072282791138, 'accumulated_submission_time': 11.330527305603027, 'accumulated_eval_time': 75.55893635749817, 'accumulated_logging_time': 0} +I0916 07:34:35.952241 140390535182080 logging_writer.py:48] [1] accumulated_eval_time=75.5589, accumulated_logging_time=0, accumulated_submission_time=11.3305, global_step=1, preemption_count=0, score=11.3305, test/accuracy=0.52922, test/loss=0.748235, test/mean_average_precision=0.0283113, test/num_examples=43793, total_duration=87.3107, train/accuracy=0.528178, train/loss=0.754217, train/mean_average_precision=0.0230337, validation/accuracy=0.528488, validation/loss=0.749869, validation/mean_average_precision=0.0271898, validation/num_examples=43793 +I0916 07:34:36.662559 140390543574784 logging_writer.py:48] [1] global_step=1, grad_norm=0.5, loss=0.756701 +I0916 07:34:36.665672 140425199580352 submission.py:307] 1) loss = 0.757, grad_norm = 0.500 +I0916 07:34:36.902549 140390535182080 logging_writer.py:48] [2] global_step=2, grad_norm=0.5, loss=0.75776 +I0916 07:34:36.905465 140425199580352 submission.py:307] 2) loss = 0.758, grad_norm = 0.500 +I0916 07:34:37.146605 140390543574784 logging_writer.py:48] [3] global_step=3, grad_norm=0.5, loss=0.757942 +I0916 07:34:37.149618 140425199580352 submission.py:307] 3) loss = 0.758, grad_norm = 0.500 +I0916 07:34:37.391516 140390535182080 logging_writer.py:48] [4] global_step=4, grad_norm=0.5, loss=0.755122 +I0916 07:34:37.394816 140425199580352 submission.py:307] 4) loss = 0.755, grad_norm = 0.500 +I0916 07:34:37.633708 140390543574784 logging_writer.py:48] [5] global_step=5, grad_norm=0.5, loss=0.752518 +I0916 07:34:37.636846 140425199580352 submission.py:307] 5) loss = 0.753, grad_norm = 0.500 +I0916 07:34:37.878890 140390535182080 logging_writer.py:48] [6] global_step=6, grad_norm=0.5, loss=0.749693 +I0916 07:34:37.882486 140425199580352 submission.py:307] 6) loss = 0.750, grad_norm = 0.500 +I0916 07:34:38.123411 140390543574784 logging_writer.py:48] [7] global_step=7, grad_norm=0.5, loss=0.745755 +I0916 07:34:38.126327 140425199580352 submission.py:307] 7) loss = 0.746, grad_norm = 0.500 +I0916 07:34:38.368447 140390535182080 logging_writer.py:48] [8] global_step=8, grad_norm=0.5, loss=0.744211 +I0916 07:34:38.371600 140425199580352 submission.py:307] 8) loss = 0.744, grad_norm = 0.500 +I0916 07:34:38.613928 140390543574784 logging_writer.py:48] [9] global_step=9, grad_norm=0.5, loss=0.738999 +I0916 07:34:38.617109 140425199580352 submission.py:307] 9) loss = 0.739, grad_norm = 0.500 +I0916 07:34:38.860470 140390535182080 logging_writer.py:48] [10] global_step=10, grad_norm=0.5, loss=0.739651 +I0916 07:34:38.863515 140425199580352 submission.py:307] 10) loss = 0.740, grad_norm = 0.500 +I0916 07:34:39.103574 140390543574784 logging_writer.py:48] [11] global_step=11, grad_norm=0.5, loss=0.735431 +I0916 07:34:39.107038 140425199580352 submission.py:307] 11) loss = 0.735, grad_norm = 0.500 +I0916 07:34:39.348393 140390535182080 logging_writer.py:48] [12] global_step=12, grad_norm=0.5, loss=0.729959 +I0916 07:34:39.352156 140425199580352 submission.py:307] 12) loss = 0.730, grad_norm = 0.500 +I0916 07:34:39.594050 140390543574784 logging_writer.py:48] [13] global_step=13, grad_norm=0.5, loss=0.721908 +I0916 07:34:39.597361 140425199580352 submission.py:307] 13) loss = 0.722, grad_norm = 0.500 +I0916 07:34:39.841251 140390535182080 logging_writer.py:48] [14] global_step=14, grad_norm=0.5, loss=0.718146 +I0916 07:34:39.844675 140425199580352 submission.py:307] 14) loss = 0.718, grad_norm = 0.500 +I0916 07:34:40.090482 140390543574784 logging_writer.py:48] [15] global_step=15, grad_norm=0.5, loss=0.712418 +I0916 07:34:40.093592 140425199580352 submission.py:307] 15) loss = 0.712, grad_norm = 0.500 +I0916 07:34:40.333024 140390535182080 logging_writer.py:48] [16] global_step=16, grad_norm=0.5, loss=0.707488 +I0916 07:34:40.335987 140425199580352 submission.py:307] 16) loss = 0.707, grad_norm = 0.500 +I0916 07:34:40.572729 140390543574784 logging_writer.py:48] [17] global_step=17, grad_norm=0.5, loss=0.701795 +I0916 07:34:40.575666 140425199580352 submission.py:307] 17) loss = 0.702, grad_norm = 0.500 +I0916 07:34:40.815145 140390535182080 logging_writer.py:48] [18] global_step=18, grad_norm=0.5, loss=0.69511 +I0916 07:34:40.818099 140425199580352 submission.py:307] 18) loss = 0.695, grad_norm = 0.500 +I0916 07:34:41.057952 140390543574784 logging_writer.py:48] [19] global_step=19, grad_norm=0.5, loss=0.687134 +I0916 07:34:41.060926 140425199580352 submission.py:307] 19) loss = 0.687, grad_norm = 0.500 +I0916 07:34:41.301592 140390535182080 logging_writer.py:48] [20] global_step=20, grad_norm=0.5, loss=0.679365 +I0916 07:34:41.304473 140425199580352 submission.py:307] 20) loss = 0.679, grad_norm = 0.500 +I0916 07:34:41.542925 140390543574784 logging_writer.py:48] [21] global_step=21, grad_norm=0.5, loss=0.67179 +I0916 07:34:41.545798 140425199580352 submission.py:307] 21) loss = 0.672, grad_norm = 0.500 +I0916 07:34:41.783957 140390535182080 logging_writer.py:48] [22] global_step=22, grad_norm=0.5, loss=0.665758 +I0916 07:34:41.786980 140425199580352 submission.py:307] 22) loss = 0.666, grad_norm = 0.500 +I0916 07:34:42.029096 140390543574784 logging_writer.py:48] [23] global_step=23, grad_norm=0.5, loss=0.659256 +I0916 07:34:42.032041 140425199580352 submission.py:307] 23) loss = 0.659, grad_norm = 0.500 +I0916 07:34:42.270804 140390535182080 logging_writer.py:48] [24] global_step=24, grad_norm=0.5, loss=0.649796 +I0916 07:34:42.273777 140425199580352 submission.py:307] 24) loss = 0.650, grad_norm = 0.500 +I0916 07:34:42.513142 140390543574784 logging_writer.py:48] [25] global_step=25, grad_norm=0.5, loss=0.644032 +I0916 07:34:42.516072 140425199580352 submission.py:307] 25) loss = 0.644, grad_norm = 0.500 +I0916 07:34:42.753794 140390535182080 logging_writer.py:48] [26] global_step=26, grad_norm=0.5, loss=0.636132 +I0916 07:34:42.757088 140425199580352 submission.py:307] 26) loss = 0.636, grad_norm = 0.500 +I0916 07:34:42.997650 140390543574784 logging_writer.py:48] [27] global_step=27, grad_norm=0.5, loss=0.628526 +I0916 07:34:43.000648 140425199580352 submission.py:307] 27) loss = 0.629, grad_norm = 0.500 +I0916 07:34:43.239491 140390535182080 logging_writer.py:48] [28] global_step=28, grad_norm=0.5, loss=0.62335 +I0916 07:34:43.243012 140425199580352 submission.py:307] 28) loss = 0.623, grad_norm = 0.500 +I0916 07:34:43.480267 140390543574784 logging_writer.py:48] [29] global_step=29, grad_norm=0.5, loss=0.615945 +I0916 07:34:43.483234 140425199580352 submission.py:307] 29) loss = 0.616, grad_norm = 0.500 +I0916 07:34:43.722530 140390535182080 logging_writer.py:48] [30] global_step=30, grad_norm=0.5, loss=0.606476 +I0916 07:34:43.725551 140425199580352 submission.py:307] 30) loss = 0.606, grad_norm = 0.500 +I0916 07:34:43.965727 140390543574784 logging_writer.py:48] [31] global_step=31, grad_norm=0.5, loss=0.599893 +I0916 07:34:43.968665 140425199580352 submission.py:307] 31) loss = 0.600, grad_norm = 0.500 +I0916 07:34:44.208460 140390535182080 logging_writer.py:48] [32] global_step=32, grad_norm=0.5, loss=0.590061 +I0916 07:34:44.211470 140425199580352 submission.py:307] 32) loss = 0.590, grad_norm = 0.500 +I0916 07:34:44.448991 140390543574784 logging_writer.py:48] [33] global_step=33, grad_norm=0.5, loss=0.584565 +I0916 07:34:44.451884 140425199580352 submission.py:307] 33) loss = 0.585, grad_norm = 0.500 +I0916 07:34:44.691999 140390535182080 logging_writer.py:48] [34] global_step=34, grad_norm=0.5, loss=0.576156 +I0916 07:34:44.695109 140425199580352 submission.py:307] 34) loss = 0.576, grad_norm = 0.500 +I0916 07:34:44.939038 140390543574784 logging_writer.py:48] [35] global_step=35, grad_norm=0.499999, loss=0.568725 +I0916 07:34:44.942005 140425199580352 submission.py:307] 35) loss = 0.569, grad_norm = 0.500 +I0916 07:34:45.182798 140390535182080 logging_writer.py:48] [36] global_step=36, grad_norm=0.499999, loss=0.563168 +I0916 07:34:45.187183 140425199580352 submission.py:307] 36) loss = 0.563, grad_norm = 0.500 +I0916 07:34:45.427038 140390543574784 logging_writer.py:48] [37] global_step=37, grad_norm=0.499999, loss=0.554549 +I0916 07:34:45.431972 140425199580352 submission.py:307] 37) loss = 0.555, grad_norm = 0.500 +I0916 07:34:45.675249 140390535182080 logging_writer.py:48] [38] global_step=38, grad_norm=0.499999, loss=0.55039 +I0916 07:34:45.678422 140425199580352 submission.py:307] 38) loss = 0.550, grad_norm = 0.500 +I0916 07:34:45.917644 140390543574784 logging_writer.py:48] [39] global_step=39, grad_norm=0.499999, loss=0.543086 +I0916 07:34:45.920676 140425199580352 submission.py:307] 39) loss = 0.543, grad_norm = 0.500 +I0916 07:34:46.156953 140390535182080 logging_writer.py:48] [40] global_step=40, grad_norm=0.499999, loss=0.537225 +I0916 07:34:46.160148 140425199580352 submission.py:307] 40) loss = 0.537, grad_norm = 0.500 +I0916 07:34:46.403579 140390543574784 logging_writer.py:48] [41] global_step=41, grad_norm=0.499999, loss=0.528337 +I0916 07:34:46.407254 140425199580352 submission.py:307] 41) loss = 0.528, grad_norm = 0.500 +I0916 07:34:46.653930 140390535182080 logging_writer.py:48] [42] global_step=42, grad_norm=0.499999, loss=0.524628 +I0916 07:34:46.657112 140425199580352 submission.py:307] 42) loss = 0.525, grad_norm = 0.500 +I0916 07:34:46.901869 140390543574784 logging_writer.py:48] [43] global_step=43, grad_norm=0.499999, loss=0.516762 +I0916 07:34:46.905040 140425199580352 submission.py:307] 43) loss = 0.517, grad_norm = 0.500 +I0916 07:34:47.145961 140390535182080 logging_writer.py:48] [44] global_step=44, grad_norm=0.499999, loss=0.50976 +I0916 07:34:47.151854 140425199580352 submission.py:307] 44) loss = 0.510, grad_norm = 0.500 +I0916 07:34:47.394057 140390543574784 logging_writer.py:48] [45] global_step=45, grad_norm=0.499999, loss=0.505603 +I0916 07:34:47.397651 140425199580352 submission.py:307] 45) loss = 0.506, grad_norm = 0.500 +I0916 07:34:47.637957 140390535182080 logging_writer.py:48] [46] global_step=46, grad_norm=0.499999, loss=0.499048 +I0916 07:34:47.641170 140425199580352 submission.py:307] 46) loss = 0.499, grad_norm = 0.500 +I0916 07:34:47.882451 140390543574784 logging_writer.py:48] [47] global_step=47, grad_norm=0.499999, loss=0.494484 +I0916 07:34:47.885479 140425199580352 submission.py:307] 47) loss = 0.494, grad_norm = 0.500 +I0916 07:34:48.123841 140390535182080 logging_writer.py:48] [48] global_step=48, grad_norm=0.499999, loss=0.490102 +I0916 07:34:48.126835 140425199580352 submission.py:307] 48) loss = 0.490, grad_norm = 0.500 +I0916 07:34:48.365929 140390543574784 logging_writer.py:48] [49] global_step=49, grad_norm=0.499999, loss=0.485355 +I0916 07:34:48.368883 140425199580352 submission.py:307] 49) loss = 0.485, grad_norm = 0.500 +I0916 07:34:48.608067 140390535182080 logging_writer.py:48] [50] global_step=50, grad_norm=0.499999, loss=0.482437 +I0916 07:34:48.611338 140425199580352 submission.py:307] 50) loss = 0.482, grad_norm = 0.500 +I0916 07:34:48.849284 140390543574784 logging_writer.py:48] [51] global_step=51, grad_norm=0.499999, loss=0.477616 +I0916 07:34:48.852277 140425199580352 submission.py:307] 51) loss = 0.478, grad_norm = 0.500 +I0916 07:34:49.091386 140390535182080 logging_writer.py:48] [52] global_step=52, grad_norm=0.499999, loss=0.471473 +I0916 07:34:49.094525 140425199580352 submission.py:307] 52) loss = 0.471, grad_norm = 0.500 +I0916 07:34:49.337899 140390543574784 logging_writer.py:48] [53] global_step=53, grad_norm=0.499999, loss=0.468569 +I0916 07:34:49.340844 140425199580352 submission.py:307] 53) loss = 0.469, grad_norm = 0.500 +I0916 07:34:49.584334 140390535182080 logging_writer.py:48] [54] global_step=54, grad_norm=0.499999, loss=0.463466 +I0916 07:34:49.587305 140425199580352 submission.py:307] 54) loss = 0.463, grad_norm = 0.500 +I0916 07:34:49.831121 140390543574784 logging_writer.py:48] [55] global_step=55, grad_norm=0.499999, loss=0.460383 +I0916 07:34:49.834061 140425199580352 submission.py:307] 55) loss = 0.460, grad_norm = 0.500 +I0916 07:34:50.074738 140390535182080 logging_writer.py:48] [56] global_step=56, grad_norm=0.499999, loss=0.454978 +I0916 07:34:50.077754 140425199580352 submission.py:307] 56) loss = 0.455, grad_norm = 0.500 +I0916 07:34:50.316841 140390543574784 logging_writer.py:48] [57] global_step=57, grad_norm=0.499999, loss=0.453774 +I0916 07:34:50.319896 140425199580352 submission.py:307] 57) loss = 0.454, grad_norm = 0.500 +I0916 07:34:50.563560 140390535182080 logging_writer.py:48] [58] global_step=58, grad_norm=0.499999, loss=0.4484 +I0916 07:34:50.566824 140425199580352 submission.py:307] 58) loss = 0.448, grad_norm = 0.500 +I0916 07:34:50.807504 140390543574784 logging_writer.py:48] [59] global_step=59, grad_norm=0.49547, loss=0.445186 +I0916 07:34:50.810398 140425199580352 submission.py:307] 59) loss = 0.445, grad_norm = 0.495 +I0916 07:34:51.054740 140390535182080 logging_writer.py:48] [60] global_step=60, grad_norm=0.492458, loss=0.441078 +I0916 07:34:51.057785 140425199580352 submission.py:307] 60) loss = 0.441, grad_norm = 0.492 +I0916 07:34:51.300877 140390543574784 logging_writer.py:48] [61] global_step=61, grad_norm=0.487307, loss=0.436396 +I0916 07:34:51.303858 140425199580352 submission.py:307] 61) loss = 0.436, grad_norm = 0.487 +I0916 07:34:51.543689 140390535182080 logging_writer.py:48] [62] global_step=62, grad_norm=0.479872, loss=0.434561 +I0916 07:34:51.546653 140425199580352 submission.py:307] 62) loss = 0.435, grad_norm = 0.480 +I0916 07:34:51.787589 140390543574784 logging_writer.py:48] [63] global_step=63, grad_norm=0.475692, loss=0.432599 +I0916 07:34:51.790627 140425199580352 submission.py:307] 63) loss = 0.433, grad_norm = 0.476 +I0916 07:34:52.032220 140390535182080 logging_writer.py:48] [64] global_step=64, grad_norm=0.476565, loss=0.428983 +I0916 07:34:52.035201 140425199580352 submission.py:307] 64) loss = 0.429, grad_norm = 0.477 +I0916 07:34:52.277191 140390543574784 logging_writer.py:48] [65] global_step=65, grad_norm=0.483333, loss=0.424186 +I0916 07:34:52.280217 140425199580352 submission.py:307] 65) loss = 0.424, grad_norm = 0.483 +I0916 07:34:52.517779 140390535182080 logging_writer.py:48] [66] global_step=66, grad_norm=0.472694, loss=0.422846 +I0916 07:34:52.520746 140425199580352 submission.py:307] 66) loss = 0.423, grad_norm = 0.473 +I0916 07:34:52.760892 140390543574784 logging_writer.py:48] [67] global_step=67, grad_norm=0.465895, loss=0.416833 +I0916 07:34:52.763900 140425199580352 submission.py:307] 67) loss = 0.417, grad_norm = 0.466 +I0916 07:34:53.005917 140390535182080 logging_writer.py:48] [68] global_step=68, grad_norm=0.459969, loss=0.416823 +I0916 07:34:53.008945 140425199580352 submission.py:307] 68) loss = 0.417, grad_norm = 0.460 +I0916 07:34:53.248730 140390543574784 logging_writer.py:48] [69] global_step=69, grad_norm=0.45264, loss=0.41346 +I0916 07:34:53.251660 140425199580352 submission.py:307] 69) loss = 0.413, grad_norm = 0.453 +I0916 07:34:53.490997 140390535182080 logging_writer.py:48] [70] global_step=70, grad_norm=0.446438, loss=0.412207 +I0916 07:34:53.494017 140425199580352 submission.py:307] 70) loss = 0.412, grad_norm = 0.446 +I0916 07:34:53.733113 140390543574784 logging_writer.py:48] [71] global_step=71, grad_norm=0.444901, loss=0.411268 +I0916 07:34:53.736115 140425199580352 submission.py:307] 71) loss = 0.411, grad_norm = 0.445 +I0916 07:34:53.975409 140390535182080 logging_writer.py:48] [72] global_step=72, grad_norm=0.439826, loss=0.408297 +I0916 07:34:53.978369 140425199580352 submission.py:307] 72) loss = 0.408, grad_norm = 0.440 +I0916 07:34:54.222257 140390543574784 logging_writer.py:48] [73] global_step=73, grad_norm=0.429563, loss=0.406558 +I0916 07:34:54.225289 140425199580352 submission.py:307] 73) loss = 0.407, grad_norm = 0.430 +I0916 07:34:54.469353 140390535182080 logging_writer.py:48] [74] global_step=74, grad_norm=0.429128, loss=0.403087 +I0916 07:34:54.472288 140425199580352 submission.py:307] 74) loss = 0.403, grad_norm = 0.429 +I0916 07:34:54.716689 140390543574784 logging_writer.py:48] [75] global_step=75, grad_norm=0.426128, loss=0.40134 +I0916 07:34:54.719737 140425199580352 submission.py:307] 75) loss = 0.401, grad_norm = 0.426 +I0916 07:34:54.962198 140390535182080 logging_writer.py:48] [76] global_step=76, grad_norm=0.423755, loss=0.398137 +I0916 07:34:54.965232 140425199580352 submission.py:307] 76) loss = 0.398, grad_norm = 0.424 +I0916 07:34:55.207509 140390543574784 logging_writer.py:48] [77] global_step=77, grad_norm=0.419658, loss=0.395656 +I0916 07:34:55.210834 140425199580352 submission.py:307] 77) loss = 0.396, grad_norm = 0.420 +I0916 07:34:55.453606 140390535182080 logging_writer.py:48] [78] global_step=78, grad_norm=0.418527, loss=0.393512 +I0916 07:34:55.456580 140425199580352 submission.py:307] 78) loss = 0.394, grad_norm = 0.419 +I0916 07:34:55.700753 140390543574784 logging_writer.py:48] [79] global_step=79, grad_norm=0.414544, loss=0.39363 +I0916 07:34:55.703722 140425199580352 submission.py:307] 79) loss = 0.394, grad_norm = 0.415 +I0916 07:34:55.947238 140390535182080 logging_writer.py:48] [80] global_step=80, grad_norm=0.415896, loss=0.388498 +I0916 07:34:55.950326 140425199580352 submission.py:307] 80) loss = 0.388, grad_norm = 0.416 +I0916 07:34:56.191327 140390543574784 logging_writer.py:48] [81] global_step=81, grad_norm=0.413136, loss=0.38738 +I0916 07:34:56.194227 140425199580352 submission.py:307] 81) loss = 0.387, grad_norm = 0.413 +I0916 07:34:56.431656 140390535182080 logging_writer.py:48] [82] global_step=82, grad_norm=0.41405, loss=0.386581 +I0916 07:34:56.434684 140425199580352 submission.py:307] 82) loss = 0.387, grad_norm = 0.414 +I0916 07:34:56.674232 140390543574784 logging_writer.py:48] [83] global_step=83, grad_norm=0.408927, loss=0.383988 +I0916 07:34:56.677589 140425199580352 submission.py:307] 83) loss = 0.384, grad_norm = 0.409 +I0916 07:34:56.917100 140390535182080 logging_writer.py:48] [84] global_step=84, grad_norm=0.408671, loss=0.382994 +I0916 07:34:56.920556 140425199580352 submission.py:307] 84) loss = 0.383, grad_norm = 0.409 +I0916 07:34:57.163501 140390543574784 logging_writer.py:48] [85] global_step=85, grad_norm=0.405117, loss=0.382958 +I0916 07:34:57.166567 140425199580352 submission.py:307] 85) loss = 0.383, grad_norm = 0.405 +I0916 07:34:57.407656 140390535182080 logging_writer.py:48] [86] global_step=86, grad_norm=0.403701, loss=0.381426 +I0916 07:34:57.410588 140425199580352 submission.py:307] 86) loss = 0.381, grad_norm = 0.404 +I0916 07:34:57.649763 140390543574784 logging_writer.py:48] [87] global_step=87, grad_norm=0.401319, loss=0.380856 +I0916 07:34:57.652763 140425199580352 submission.py:307] 87) loss = 0.381, grad_norm = 0.401 +I0916 07:34:57.892662 140390535182080 logging_writer.py:48] [88] global_step=88, grad_norm=0.4012, loss=0.375607 +I0916 07:34:57.895662 140425199580352 submission.py:307] 88) loss = 0.376, grad_norm = 0.401 +I0916 07:34:58.135431 140390543574784 logging_writer.py:48] [89] global_step=89, grad_norm=0.394719, loss=0.378277 +I0916 07:34:58.138400 140425199580352 submission.py:307] 89) loss = 0.378, grad_norm = 0.395 +I0916 07:34:58.380790 140390535182080 logging_writer.py:48] [90] global_step=90, grad_norm=0.398387, loss=0.373105 +I0916 07:34:58.384269 140425199580352 submission.py:307] 90) loss = 0.373, grad_norm = 0.398 +I0916 07:34:58.623115 140390543574784 logging_writer.py:48] [91] global_step=91, grad_norm=0.399344, loss=0.370765 +I0916 07:34:58.626153 140425199580352 submission.py:307] 91) loss = 0.371, grad_norm = 0.399 +I0916 07:34:58.865088 140390535182080 logging_writer.py:48] [92] global_step=92, grad_norm=0.394602, loss=0.371645 +I0916 07:34:58.868099 140425199580352 submission.py:307] 92) loss = 0.372, grad_norm = 0.395 +I0916 07:34:59.108284 140390543574784 logging_writer.py:48] [93] global_step=93, grad_norm=0.392345, loss=0.368974 +I0916 07:34:59.111214 140425199580352 submission.py:307] 93) loss = 0.369, grad_norm = 0.392 +I0916 07:34:59.347743 140390535182080 logging_writer.py:48] [94] global_step=94, grad_norm=0.392425, loss=0.364885 +I0916 07:34:59.350697 140425199580352 submission.py:307] 94) loss = 0.365, grad_norm = 0.392 +I0916 07:34:59.592038 140390543574784 logging_writer.py:48] [95] global_step=95, grad_norm=0.390491, loss=0.367709 +I0916 07:34:59.595041 140425199580352 submission.py:307] 95) loss = 0.368, grad_norm = 0.390 +I0916 07:34:59.832926 140390535182080 logging_writer.py:48] [96] global_step=96, grad_norm=0.386259, loss=0.367995 +I0916 07:34:59.835960 140425199580352 submission.py:307] 96) loss = 0.368, grad_norm = 0.386 +I0916 07:35:00.079825 140390543574784 logging_writer.py:48] [97] global_step=97, grad_norm=0.386014, loss=0.36332 +I0916 07:35:00.082894 140425199580352 submission.py:307] 97) loss = 0.363, grad_norm = 0.386 +I0916 07:35:00.329309 140390535182080 logging_writer.py:48] [98] global_step=98, grad_norm=0.386592, loss=0.359605 +I0916 07:35:00.332296 140425199580352 submission.py:307] 98) loss = 0.360, grad_norm = 0.387 +I0916 07:35:00.574132 140390543574784 logging_writer.py:48] [99] global_step=99, grad_norm=0.381879, loss=0.362913 +I0916 07:35:00.577128 140425199580352 submission.py:307] 99) loss = 0.363, grad_norm = 0.382 +I0916 07:35:00.816957 140390535182080 logging_writer.py:48] [100] global_step=100, grad_norm=0.385491, loss=0.355971 +I0916 07:35:00.820179 140425199580352 submission.py:307] 100) loss = 0.356, grad_norm = 0.385 +I0916 07:36:34.030071 140390543574784 logging_writer.py:48] [500] global_step=500, grad_norm=0.0334578, loss=0.0671956 +I0916 07:36:34.034708 140425199580352 submission.py:307] 500) loss = 0.067, grad_norm = 0.033 +I0916 07:38:30.922366 140390535182080 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.0306246, loss=0.0568154 +I0916 07:38:30.926072 140425199580352 submission.py:307] 1000) loss = 0.057, grad_norm = 0.031 +I0916 07:40:27.343349 140390543574784 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.0174965, loss=0.0472846 +I0916 07:40:27.347029 140425199580352 submission.py:307] 1500) loss = 0.047, grad_norm = 0.017 +I0916 07:42:08.569755 140425199580352 spec.py:333] Evaluating on the training split. +I0916 07:42:34.633372 140425199580352 spec.py:346] Evaluating on the validation split. +I0916 07:42:36.681307 140425199580352 spec.py:363] Evaluating on the test split. +I0916 07:42:38.708200 140425199580352 submission_runner.py:516] Time since start: 570.09s, Step: 1934, {'train/accuracy': 0.9872739783359699, 'train/loss': 0.04662817519994262, 'train/mean_average_precision': 0.10350142962238218, 'validation/accuracy': 0.9846617349540007, 'validation/loss': 0.055813864515374796, 'validation/mean_average_precision': 0.10950719332545168, 'validation/num_examples': 43793, 'test/accuracy': 0.983643984067235, 'test/loss': 0.05911211064073872, 'test/mean_average_precision': 0.10651275554585107, 'test/num_examples': 43793, 'score': 462.54997301101685, 'total_duration': 570.088809967041, 'accumulated_submission_time': 462.54997301101685, 'accumulated_eval_time': 105.69732356071472, 'accumulated_logging_time': 0.032109737396240234} +I0916 07:42:38.733559 140390535182080 logging_writer.py:48] [1934] accumulated_eval_time=105.697, accumulated_logging_time=0.0321097, accumulated_submission_time=462.55, global_step=1934, preemption_count=0, score=462.55, test/accuracy=0.983644, test/loss=0.0591121, test/mean_average_precision=0.106513, test/num_examples=43793, total_duration=570.089, train/accuracy=0.987274, train/loss=0.0466282, train/mean_average_precision=0.103501, validation/accuracy=0.984662, validation/loss=0.0558139, validation/mean_average_precision=0.109507, validation/num_examples=43793 +I0916 07:42:54.722208 140390543574784 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.0344604, loss=0.0416965 +I0916 07:42:54.725249 140425199580352 submission.py:307] 2000) loss = 0.042, grad_norm = 0.034 +I0916 07:44:51.181330 140390535182080 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.0239974, loss=0.0423151 +I0916 07:44:51.184902 140425199580352 submission.py:307] 2500) loss = 0.042, grad_norm = 0.024 +I0916 07:46:48.243420 140390543574784 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.0409143, loss=0.0434758 +I0916 07:46:48.246855 140425199580352 submission.py:307] 3000) loss = 0.043, grad_norm = 0.041 +I0916 07:48:45.003626 140390535182080 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.0222061, loss=0.0420215 +I0916 07:48:45.007270 140425199580352 submission.py:307] 3500) loss = 0.042, grad_norm = 0.022 +I0916 07:50:11.338231 140425199580352 spec.py:333] Evaluating on the training split. +I0916 07:50:36.790089 140425199580352 spec.py:346] Evaluating on the validation split. +I0916 07:50:38.799749 140425199580352 spec.py:363] Evaluating on the test split. +I0916 07:50:40.849383 140425199580352 submission_runner.py:516] Time since start: 1052.23s, Step: 3872, {'train/accuracy': 0.988466569277978, 'train/loss': 0.040134937259917125, 'train/mean_average_precision': 0.19431109139602815, 'validation/accuracy': 0.9854650596384349, 'validation/loss': 0.0498583678504509, 'validation/mean_average_precision': 0.16580154988068696, 'validation/num_examples': 43793, 'test/accuracy': 0.9845637651728085, 'test/loss': 0.05259035814924375, 'test/mean_average_precision': 0.16993341334131962, 'test/num_examples': 43793, 'score': 913.7683036327362, 'total_duration': 1052.229915857315, 'accumulated_submission_time': 913.7683036327362, 'accumulated_eval_time': 135.20834755897522, 'accumulated_logging_time': 0.06734704971313477} +I0916 07:50:40.872758 140390543574784 logging_writer.py:48] [3872] accumulated_eval_time=135.208, accumulated_logging_time=0.067347, accumulated_submission_time=913.768, global_step=3872, preemption_count=0, score=913.768, test/accuracy=0.984564, test/loss=0.0525904, test/mean_average_precision=0.169933, test/num_examples=43793, total_duration=1052.23, train/accuracy=0.988467, train/loss=0.0401349, train/mean_average_precision=0.194311, validation/accuracy=0.985465, validation/loss=0.0498584, validation/mean_average_precision=0.165802, validation/num_examples=43793 +I0916 07:51:11.006477 140390535182080 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.0219006, loss=0.0391567 +I0916 07:51:11.009632 140425199580352 submission.py:307] 4000) loss = 0.039, grad_norm = 0.022 +I0916 07:53:05.453894 140390543574784 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.0183786, loss=0.040737 +I0916 07:53:05.457531 140425199580352 submission.py:307] 4500) loss = 0.041, grad_norm = 0.018 +I0916 07:55:00.315978 140390535182080 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.0348569, loss=0.037563 +I0916 07:55:00.319338 140425199580352 submission.py:307] 5000) loss = 0.038, grad_norm = 0.035 +I0916 07:56:54.691766 140390543574784 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.0195123, loss=0.04068 +I0916 07:56:54.695199 140425199580352 submission.py:307] 5500) loss = 0.041, grad_norm = 0.020 +I0916 07:58:13.359514 140425199580352 spec.py:333] Evaluating on the training split. +I0916 07:58:38.934365 140425199580352 spec.py:346] Evaluating on the validation split. +I0916 07:58:40.968616 140425199580352 spec.py:363] Evaluating on the test split. +I0916 07:58:42.997476 140425199580352 submission_runner.py:516] Time since start: 1534.38s, Step: 5844, {'train/accuracy': 0.9888896233378502, 'train/loss': 0.03787011370942104, 'train/mean_average_precision': 0.2467587829208071, 'validation/accuracy': 0.985922333644001, 'validation/loss': 0.04776651813549112, 'validation/mean_average_precision': 0.19920993286886418, 'validation/num_examples': 43793, 'test/accuracy': 0.9850217605722184, 'test/loss': 0.050569384355214075, 'test/mean_average_precision': 0.20027314266106533, 'test/num_examples': 43793, 'score': 1364.8587460517883, 'total_duration': 1534.3781569004059, 'accumulated_submission_time': 1364.8587460517883, 'accumulated_eval_time': 164.8463044166565, 'accumulated_logging_time': 0.10014581680297852} +I0916 07:58:43.023336 140390535182080 logging_writer.py:48] [5844] accumulated_eval_time=164.846, accumulated_logging_time=0.100146, accumulated_submission_time=1364.86, global_step=5844, preemption_count=0, score=1364.86, test/accuracy=0.985022, test/loss=0.0505694, test/mean_average_precision=0.200273, test/num_examples=43793, total_duration=1534.38, train/accuracy=0.98889, train/loss=0.0378701, train/mean_average_precision=0.246759, validation/accuracy=0.985922, validation/loss=0.0477665, validation/mean_average_precision=0.19921, validation/num_examples=43793 +I0916 07:59:19.329839 140390543574784 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.0247361, loss=0.0396489 +I0916 07:59:19.332813 140425199580352 submission.py:307] 6000) loss = 0.040, grad_norm = 0.025 +I0916 08:01:13.308785 140390535182080 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.0183451, loss=0.0359333 +I0916 08:01:13.312162 140425199580352 submission.py:307] 6500) loss = 0.036, grad_norm = 0.018 +I0916 08:03:07.632609 140390543574784 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.0148427, loss=0.0402458 +I0916 08:03:07.636105 140425199580352 submission.py:307] 7000) loss = 0.040, grad_norm = 0.015 +I0916 08:05:02.116398 140390535182080 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.0189646, loss=0.040157 +I0916 08:05:02.119786 140425199580352 submission.py:307] 7500) loss = 0.040, grad_norm = 0.019 +I0916 08:06:15.512043 140425199580352 spec.py:333] Evaluating on the training split. +I0916 08:06:40.807269 140425199580352 spec.py:346] Evaluating on the validation split. +I0916 08:06:42.837552 140425199580352 spec.py:363] Evaluating on the test split. +I0916 08:06:44.893891 140425199580352 submission_runner.py:516] Time since start: 2016.27s, Step: 7820, {'train/accuracy': 0.9895427617196845, 'train/loss': 0.035386638148443644, 'train/mean_average_precision': 0.2761431838430722, 'validation/accuracy': 0.9862570184354693, 'validation/loss': 0.04642729059164315, 'validation/mean_average_precision': 0.22400625690558534, 'validation/num_examples': 43793, 'test/accuracy': 0.9853462529448578, 'test/loss': 0.04926592023594238, 'test/mean_average_precision': 0.21944701857998322, 'test/num_examples': 43793, 'score': 1815.9603216648102, 'total_duration': 2016.2744965553284, 'accumulated_submission_time': 1815.9603216648102, 'accumulated_eval_time': 194.22807431221008, 'accumulated_logging_time': 0.13584184646606445} +I0916 08:06:44.916714 140390543574784 logging_writer.py:48] [7820] accumulated_eval_time=194.228, accumulated_logging_time=0.135842, accumulated_submission_time=1815.96, global_step=7820, preemption_count=0, score=1815.96, test/accuracy=0.985346, test/loss=0.0492659, test/mean_average_precision=0.219447, test/num_examples=43793, total_duration=2016.27, train/accuracy=0.989543, train/loss=0.0353866, train/mean_average_precision=0.276143, validation/accuracy=0.986257, validation/loss=0.0464273, validation/mean_average_precision=0.224006, validation/num_examples=43793 +I0916 08:07:26.801909 140390535182080 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.0148677, loss=0.0351541 +I0916 08:07:26.804994 140425199580352 submission.py:307] 8000) loss = 0.035, grad_norm = 0.015 +I0916 08:09:20.633471 140390543574784 logging_writer.py:48] [8500] global_step=8500, grad_norm=0.0265448, loss=0.0358589 +I0916 08:09:20.637291 140425199580352 submission.py:307] 8500) loss = 0.036, grad_norm = 0.027 +I0916 08:11:14.683550 140390535182080 logging_writer.py:48] [9000] global_step=9000, grad_norm=0.0226706, loss=0.0407664 +I0916 08:11:14.687241 140425199580352 submission.py:307] 9000) loss = 0.041, grad_norm = 0.023 +I0916 08:13:09.111988 140390543574784 logging_writer.py:48] [9500] global_step=9500, grad_norm=0.0184488, loss=0.0375356 +I0916 08:13:09.115563 140425199580352 submission.py:307] 9500) loss = 0.038, grad_norm = 0.018 +I0916 08:14:17.368171 140425199580352 spec.py:333] Evaluating on the training split. +I0916 08:14:42.811419 140425199580352 spec.py:346] Evaluating on the validation split. +I0916 08:14:44.841966 140425199580352 spec.py:363] Evaluating on the test split. +I0916 08:14:46.905890 140425199580352 submission_runner.py:516] Time since start: 2498.29s, Step: 9799, {'train/accuracy': 0.9901857675683852, 'train/loss': 0.03309017166229622, 'train/mean_average_precision': 0.3490473628829426, 'validation/accuracy': 0.9865009790900146, 'validation/loss': 0.0451063550228292, 'validation/mean_average_precision': 0.24864923532023483, 'validation/num_examples': 43793, 'test/accuracy': 0.9857101223932165, 'test/loss': 0.04764598195371783, 'test/mean_average_precision': 0.2409842598333483, 'test/num_examples': 43793, 'score': 2267.0146951675415, 'total_duration': 2498.28653550148, 'accumulated_submission_time': 2267.0146951675415, 'accumulated_eval_time': 223.76578831672668, 'accumulated_logging_time': 0.16794610023498535} +I0916 08:14:46.928147 140390535182080 logging_writer.py:48] [9799] accumulated_eval_time=223.766, accumulated_logging_time=0.167946, accumulated_submission_time=2267.01, global_step=9799, preemption_count=0, score=2267.01, test/accuracy=0.98571, test/loss=0.047646, test/mean_average_precision=0.240984, test/num_examples=43793, total_duration=2498.29, train/accuracy=0.990186, train/loss=0.0330902, train/mean_average_precision=0.349047, validation/accuracy=0.986501, validation/loss=0.0451064, validation/mean_average_precision=0.248649, validation/num_examples=43793 +I0916 08:15:33.728792 140390543574784 logging_writer.py:48] [10000] global_step=10000, grad_norm=0.0164723, loss=0.0345669 +I0916 08:15:33.732201 140425199580352 submission.py:307] 10000) loss = 0.035, grad_norm = 0.016 +I0916 08:17:27.625072 140390535182080 logging_writer.py:48] [10500] global_step=10500, grad_norm=0.0197831, loss=0.0361742 +I0916 08:17:27.628412 140425199580352 submission.py:307] 10500) loss = 0.036, grad_norm = 0.020 +I0916 08:19:21.081418 140390543574784 logging_writer.py:48] [11000] global_step=11000, grad_norm=0.0182838, loss=0.0359306 +I0916 08:19:21.084796 140425199580352 submission.py:307] 11000) loss = 0.036, grad_norm = 0.018 +I0916 08:21:15.665535 140390535182080 logging_writer.py:48] [11500] global_step=11500, grad_norm=0.0167634, loss=0.0306401 +I0916 08:21:15.668851 140425199580352 submission.py:307] 11500) loss = 0.031, grad_norm = 0.017 +I0916 08:22:19.406543 140425199580352 spec.py:333] Evaluating on the training split. +I0916 08:22:44.875385 140425199580352 spec.py:346] Evaluating on the validation split. +I0916 08:22:46.885743 140425199580352 spec.py:363] Evaluating on the test split. +I0916 08:22:48.914321 140425199580352 submission_runner.py:516] Time since start: 2980.29s, Step: 11778, {'train/accuracy': 0.9903584234755127, 'train/loss': 0.03182477578347286, 'train/mean_average_precision': 0.3719740073263724, 'validation/accuracy': 0.986685268835711, 'validation/loss': 0.04487150230037443, 'validation/mean_average_precision': 0.25828172536038974, 'validation/num_examples': 43793, 'test/accuracy': 0.9858520483238843, 'test/loss': 0.04761673539405292, 'test/mean_average_precision': 0.2583378703119949, 'test/num_examples': 43793, 'score': 2718.1068575382233, 'total_duration': 2980.2949731349945, 'accumulated_submission_time': 2718.1068575382233, 'accumulated_eval_time': 253.27358293533325, 'accumulated_logging_time': 0.19945549964904785} +I0916 08:22:48.937652 140390543574784 logging_writer.py:48] [11778] accumulated_eval_time=253.274, accumulated_logging_time=0.199455, accumulated_submission_time=2718.11, global_step=11778, preemption_count=0, score=2718.11, test/accuracy=0.985852, test/loss=0.0476167, test/mean_average_precision=0.258338, test/num_examples=43793, total_duration=2980.29, train/accuracy=0.990358, train/loss=0.0318248, train/mean_average_precision=0.371974, validation/accuracy=0.986685, validation/loss=0.0448715, validation/mean_average_precision=0.258282, validation/num_examples=43793 +I0916 08:23:40.402677 140390535182080 logging_writer.py:48] [12000] global_step=12000, grad_norm=0.0158963, loss=0.0346977 +I0916 08:23:40.406143 140425199580352 submission.py:307] 12000) loss = 0.035, grad_norm = 0.016 +I0916 08:25:33.943518 140390543574784 logging_writer.py:48] [12500] global_step=12500, grad_norm=0.0168645, loss=0.0319809 +I0916 08:25:33.946823 140425199580352 submission.py:307] 12500) loss = 0.032, grad_norm = 0.017 +I0916 08:27:28.525973 140390535182080 logging_writer.py:48] [13000] global_step=13000, grad_norm=0.0163647, loss=0.0322085 +I0916 08:27:28.531259 140425199580352 submission.py:307] 13000) loss = 0.032, grad_norm = 0.016 +I0916 08:29:23.150462 140390543574784 logging_writer.py:48] [13500] global_step=13500, grad_norm=0.0162139, loss=0.0314379 +I0916 08:29:23.154107 140425199580352 submission.py:307] 13500) loss = 0.031, grad_norm = 0.016 +I0916 08:30:21.352292 140425199580352 spec.py:333] Evaluating on the training split. +I0916 08:30:46.917466 140425199580352 spec.py:346] Evaluating on the validation split. +I0916 08:30:48.933404 140425199580352 spec.py:363] Evaluating on the test split. +I0916 08:30:50.978835 140425199580352 submission_runner.py:516] Time since start: 3462.36s, Step: 13753, {'train/accuracy': 0.9909471630031864, 'train/loss': 0.030043816366344483, 'train/mean_average_precision': 0.4012493489435839, 'validation/accuracy': 0.9867646270852344, 'validation/loss': 0.04451309946430137, 'validation/mean_average_precision': 0.2676612510009205, 'validation/num_examples': 43793, 'test/accuracy': 0.9860992605532669, 'test/loss': 0.046987406284665516, 'test/mean_average_precision': 0.26131275504819856, 'test/num_examples': 43793, 'score': 3169.12438416481, 'total_duration': 3462.3594183921814, 'accumulated_submission_time': 3169.12438416481, 'accumulated_eval_time': 282.9000051021576, 'accumulated_logging_time': 0.2322547435760498} +I0916 08:30:51.004554 140390535182080 logging_writer.py:48] [13753] accumulated_eval_time=282.9, accumulated_logging_time=0.232255, accumulated_submission_time=3169.12, global_step=13753, preemption_count=0, score=3169.12, test/accuracy=0.986099, test/loss=0.0469874, test/mean_average_precision=0.261313, test/num_examples=43793, total_duration=3462.36, train/accuracy=0.990947, train/loss=0.0300438, train/mean_average_precision=0.401249, validation/accuracy=0.986765, validation/loss=0.0445131, validation/mean_average_precision=0.267661, validation/num_examples=43793 +I0916 08:31:48.064562 140390543574784 logging_writer.py:48] [14000] global_step=14000, grad_norm=0.0175456, loss=0.0326298 +I0916 08:31:48.072448 140425199580352 submission.py:307] 14000) loss = 0.033, grad_norm = 0.018 +I0916 08:33:42.248292 140390535182080 logging_writer.py:48] [14500] global_step=14500, grad_norm=0.0180858, loss=0.0315818 +I0916 08:33:42.251706 140425199580352 submission.py:307] 14500) loss = 0.032, grad_norm = 0.018 +I0916 08:35:35.850960 140390543574784 logging_writer.py:48] [15000] global_step=15000, grad_norm=0.0207067, loss=0.0332999 +I0916 08:35:35.855293 140425199580352 submission.py:307] 15000) loss = 0.033, grad_norm = 0.021 +I0916 08:37:29.566195 140390535182080 logging_writer.py:48] [15500] global_step=15500, grad_norm=0.0167546, loss=0.0302563 +I0916 08:37:29.576124 140425199580352 submission.py:307] 15500) loss = 0.030, grad_norm = 0.017 +I0916 08:38:23.515446 140425199580352 spec.py:333] Evaluating on the training split. +I0916 08:38:48.829269 140425199580352 spec.py:346] Evaluating on the validation split. +I0916 08:38:50.814771 140425199580352 spec.py:363] Evaluating on the test split. +I0916 08:38:52.785982 140425199580352 submission_runner.py:516] Time since start: 3944.17s, Step: 15735, {'train/accuracy': 0.9914554525443318, 'train/loss': 0.02781956239430765, 'train/mean_average_precision': 0.46531210795072864, 'validation/accuracy': 0.9869414072267559, 'validation/loss': 0.04474040135586918, 'validation/mean_average_precision': 0.2762624088256628, 'validation/num_examples': 43793, 'test/accuracy': 0.9861727503897699, 'test/loss': 0.04756114422833145, 'test/mean_average_precision': 0.2682699677631099, 'test/num_examples': 43793, 'score': 3620.244943380356, 'total_duration': 3944.1666402816772, 'accumulated_submission_time': 3620.244943380356, 'accumulated_eval_time': 312.17052459716797, 'accumulated_logging_time': 0.267136812210083} +I0916 08:38:52.809207 140390543574784 logging_writer.py:48] [15735] accumulated_eval_time=312.171, accumulated_logging_time=0.267137, accumulated_submission_time=3620.24, global_step=15735, preemption_count=0, score=3620.24, test/accuracy=0.986173, test/loss=0.0475611, test/mean_average_precision=0.26827, test/num_examples=43793, total_duration=3944.17, train/accuracy=0.991455, train/loss=0.0278196, train/mean_average_precision=0.465312, validation/accuracy=0.986941, validation/loss=0.0447404, validation/mean_average_precision=0.276262, validation/num_examples=43793 +I0916 08:39:54.301081 140390535182080 logging_writer.py:48] [16000] global_step=16000, grad_norm=0.0183285, loss=0.0314843 +I0916 08:39:54.304349 140425199580352 submission.py:307] 16000) loss = 0.031, grad_norm = 0.018 +I0916 08:41:48.900635 140390543574784 logging_writer.py:48] [16500] global_step=16500, grad_norm=0.0170706, loss=0.0278501 +I0916 08:41:48.904159 140425199580352 submission.py:307] 16500) loss = 0.028, grad_norm = 0.017 +I0916 08:43:43.054923 140390535182080 logging_writer.py:48] [17000] global_step=17000, grad_norm=0.0183626, loss=0.0295564 +I0916 08:43:43.058679 140425199580352 submission.py:307] 17000) loss = 0.030, grad_norm = 0.018 +I0916 08:45:37.236182 140390543574784 logging_writer.py:48] [17500] global_step=17500, grad_norm=0.0158082, loss=0.0270338 +I0916 08:45:37.242890 140425199580352 submission.py:307] 17500) loss = 0.027, grad_norm = 0.016 +I0916 08:46:25.219479 140425199580352 spec.py:333] Evaluating on the training split. +I0916 08:46:50.526347 140425199580352 spec.py:346] Evaluating on the validation split. +I0916 08:46:52.521370 140425199580352 spec.py:363] Evaluating on the test split. +I0916 08:46:54.502580 140425199580352 submission_runner.py:516] Time since start: 4425.88s, Step: 17711, {'train/accuracy': 0.9920030392661061, 'train/loss': 0.02643663386916597, 'train/mean_average_precision': 0.4929768841281881, 'validation/accuracy': 0.9868512919766577, 'validation/loss': 0.04476370713335271, 'validation/mean_average_precision': 0.2784861725845394, 'validation/num_examples': 43793, 'test/accuracy': 0.9860162949498795, 'test/loss': 0.04755494549749461, 'test/mean_average_precision': 0.26342184124267576, 'test/num_examples': 43793, 'score': 4071.263598918915, 'total_duration': 4425.883248329163, 'accumulated_submission_time': 4071.263598918915, 'accumulated_eval_time': 341.4536712169647, 'accumulated_logging_time': 0.3021092414855957} +I0916 08:46:54.526259 140390535182080 logging_writer.py:48] [17711] accumulated_eval_time=341.454, accumulated_logging_time=0.302109, accumulated_submission_time=4071.26, global_step=17711, preemption_count=0, score=4071.26, test/accuracy=0.986016, test/loss=0.0475549, test/mean_average_precision=0.263422, test/num_examples=43793, total_duration=4425.88, train/accuracy=0.992003, train/loss=0.0264366, train/mean_average_precision=0.492977, validation/accuracy=0.986851, validation/loss=0.0447637, validation/mean_average_precision=0.278486, validation/num_examples=43793 +I0916 08:48:00.610107 140390543574784 logging_writer.py:48] [18000] global_step=18000, grad_norm=0.0210767, loss=0.0288594 +I0916 08:48:00.613471 140425199580352 submission.py:307] 18000) loss = 0.029, grad_norm = 0.021 +I0916 08:49:53.865504 140390535182080 logging_writer.py:48] [18500] global_step=18500, grad_norm=0.017593, loss=0.0279048 +I0916 08:49:53.868915 140425199580352 submission.py:307] 18500) loss = 0.028, grad_norm = 0.018 +I0916 08:51:46.730451 140390543574784 logging_writer.py:48] [19000] global_step=19000, grad_norm=0.0201181, loss=0.0298432 +I0916 08:51:46.733801 140425199580352 submission.py:307] 19000) loss = 0.030, grad_norm = 0.020 +I0916 08:53:39.242175 140390535182080 logging_writer.py:48] [19500] global_step=19500, grad_norm=0.0175766, loss=0.0273489 +I0916 08:53:39.245586 140425199580352 submission.py:307] 19500) loss = 0.027, grad_norm = 0.018 +I0916 08:54:27.014049 140425199580352 spec.py:333] Evaluating on the training split. +I0916 08:54:52.518085 140425199580352 spec.py:346] Evaluating on the validation split. +I0916 08:54:54.513068 140425199580352 spec.py:363] Evaluating on the test split. +I0916 08:54:56.497677 140425199580352 submission_runner.py:516] Time since start: 4907.88s, Step: 19711, {'train/accuracy': 0.9921952268836711, 'train/loss': 0.025366222333885333, 'train/mean_average_precision': 0.5251250965686298, 'validation/accuracy': 0.9868525097503077, 'validation/loss': 0.04500548277215617, 'validation/mean_average_precision': 0.28200299588380895, 'validation/num_examples': 43793, 'test/accuracy': 0.9859518597350659, 'test/loss': 0.04796572657232453, 'test/mean_average_precision': 0.2710933780442979, 'test/num_examples': 43793, 'score': 4522.396051645279, 'total_duration': 4907.878260135651, 'accumulated_submission_time': 4522.396051645279, 'accumulated_eval_time': 370.93720149993896, 'accumulated_logging_time': 0.33507418632507324} +I0916 08:54:56.521176 140390543574784 logging_writer.py:48] [19711] accumulated_eval_time=370.937, accumulated_logging_time=0.335074, accumulated_submission_time=4522.4, global_step=19711, preemption_count=0, score=4522.4, test/accuracy=0.985952, test/loss=0.0479657, test/mean_average_precision=0.271093, test/num_examples=43793, total_duration=4907.88, train/accuracy=0.992195, train/loss=0.0253662, train/mean_average_precision=0.525125, validation/accuracy=0.986853, validation/loss=0.0450055, validation/mean_average_precision=0.282003, validation/num_examples=43793 +I0916 08:54:56.965873 140390535182080 logging_writer.py:48] [19711] global_step=19711, preemption_count=0, score=4522.4 +I0916 08:54:57.093461 140425199580352 submission_runner.py:857] Final ogbg score: 4522.396051645279 diff --git a/logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..a3805088 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,12 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/accuracy,test/loss,test/mean_average_precision,test/num_examples,total_duration,train/accuracy,train/loss,train/mean_average_precision,validation/accuracy,validation/loss,validation/mean_average_precision,validation/num_examples +75.55893635749817,0.0,11.330527305603027,1,0,11.330527305603027,0.5292198959097536,0.7482351910609405,0.0283112904389152,43793,87.31072282791138,0.5281778531601953,0.7542170812830864,0.0230337421923166,0.5284877849184416,0.7498691400731963,0.0271897867873256,43793 +105.69732356071472,0.0321097373962402,462.5499730110169,1934,0,462.5499730110169,0.983643984067235,0.0591121106407387,0.106512755545851,43793,570.088809967041,0.98727397833597,0.0466281751999426,0.1035014296223821,0.9846617349540008,0.0558138645153747,0.1095071933254516,43793 +135.20834755897522,0.0673470497131347,913.7683036327362,3872,0,913.7683036327362,0.9845637651728084,0.0525903581492437,0.1699334133413196,43793,1052.229915857315,0.988466569277978,0.0401349372599171,0.1943110913960281,0.9854650596384348,0.0498583678504509,0.1658015498806869,43793 +164.8463044166565,0.1001458168029785,1364.8587460517883,5844,0,1364.8587460517883,0.9850217605722184,0.050569384355214,0.2002731426610653,43793,1534.378156900406,0.9888896233378502,0.037870113709421,0.2467587829208071,0.985922333644001,0.0477665181354911,0.1992099328688641,43793 +194.22807431221008,0.1358418464660644,1815.9603216648104,7820,0,1815.9603216648104,0.9853462529448578,0.0492659202359423,0.2194470185799832,43793,2016.2744965553284,0.9895427617196844,0.0353866381484436,0.2761431838430722,0.9862570184354692,0.0464272905916431,0.2240062569055853,43793 +223.76578831672668,0.1679461002349853,2267.014695167541,9799,0,2267.014695167541,0.9857101223932164,0.0476459819537178,0.2409842598333483,43793,2498.28653550148,0.9901857675683852,0.0330901716622962,0.3490473628829426,0.9865009790900146,0.0451063550228292,0.2486492353202348,43793 +253.27358293533325,0.1994554996490478,2718.106857538224,11778,0,2718.106857538224,0.9858520483238844,0.0476167353940529,0.2583378703119949,43793,2980.2949731349945,0.9903584234755128,0.0318247757834728,0.3719740073263724,0.986685268835711,0.0448715023003744,0.2582817253603897,43793 +282.9000051021576,0.2322547435760498,3169.12438416481,13753,0,3169.12438416481,0.9860992605532668,0.0469874062846655,0.2613127550481985,43793,3462.3594183921814,0.9909471630031864,0.0300438163663444,0.4012493489435839,0.9867646270852344,0.0445130994643013,0.2676612510009205,43793 +312.170524597168,0.267136812210083,3620.244943380356,15735,0,3620.244943380356,0.98617275038977,0.0475611442283314,0.2682699677631099,43793,3944.166640281677,0.9914554525443318,0.0278195623943076,0.4653121079507286,0.986941407226756,0.0447404013558691,0.2762624088256628,43793 +341.4536712169647,0.3021092414855957,4071.263598918915,17711,0,4071.263598918915,0.9860162949498796,0.0475549454974946,0.2634218412426757,43793,4425.883248329163,0.992003039266106,0.0264366338691659,0.4929768841281881,0.9868512919766576,0.0447637071333527,0.2784861725845394,43793 +370.93720149993896,0.33507418632507324,4522.396051645279,19711,0,4522.396051645279,0.9859518597350659,0.04796572657232453,0.2710933780442979,43793,4907.878260135651,0.9921952268836711,0.025366222333885333,0.5251250965686298,0.9868525097503077,0.04500548277215617,0.28200299588380895,43793 diff --git a/logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/trial_1/events.out.tfevents.1789543988.791f03d5f744.38.0.v2 b/logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/trial_1/events.out.tfevents.1789543988.791f03d5f744.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..822b3e88eb8794485e360d311d40ba8444e050c7 GIT binary patch literal 43428 zcmbW=d00*F-Ujdt$=-=3b0LaK5*b2Kl$nf?DU~EjX(S>k%2bicOe$oaLntyNLP*9! zRLERNagM$6Jo{clcE9u2dtK-K-q-p(&l>Kwz1Qh0$NcAEwt1oG|NrNj$fQRWEtDKQ z?Oo;?TKPK7_3qtujJCL~x6@oNch8xw?tX@j?w(GDKHe^_hE9L}doMdbPZw_|PdWGA zB7>^5uLo~fiGFKWg#Ss?*AllD*L8LG^6D?HUrSZ4ZQX5Y7X5}QibY}tFGqVJVK>*^)9r7-_us$!LO@jH&PwB|fCjtU*oe07 ztP0Q-w-+lqySh8ryV`kqJ9+$#`1Ah^5mP&c8=6!_>{<{oL1fXWGVaguE7#KRDAp4< z^!~FZo$VbReLU?Q7yQjh>Y;#^iTl$&cB{&nX54g&=xcM5(?P5w7Be~0>tUzPT92om zi!=vTWq5W!ogh+fOEP+idx|xf3^ynHxpwxxPM-G8PIexiPL3{KF79*x9$Eh*DW)L& z5w=9ztSWW-kWcSKXZ1*`k=RhI=4fAE~Pt5%^(4s0@ zZtul?qW%^nx|_JGxT#e1Tpu?(r}_479XUQ&qbrP{PKx=&STi+1~`i0)4%p@w1uv5F*Arj;^_)BfJMD#~rD z<|vWXOcK>r+(s<#_2)M7_f&l_q`L(yv|8TCg>W%$aU z#mfJ`;W)W@{8@fGM|YpO-hVfNb2$5ls{0#OC0)*(P%iRYs=ur1&dPZ@o+|W6(MhA@h24@o&8s6dfM*VidKxPqNh~V#Y#BveuHv{gYcJAHXnKZ3 z2rG7=;*PGA#Wj)8%z5G%PiqKhPT1G{3y`s&a80JYLgcYf%n3|}1 z1{C3yD;R|WS8=nrFYZNA{Hi;kxFa9>cbZ?pxh_KO8&3j7?S>>mSdotk*PzuWK@oEx zhB=P35xl8rx=Bpt2k%NQdkAO zi2_BeF`4F9aQ0%(?_?`byy!zBgcWU2@mOWuR!~%?U11c8TtzP{Q{0PDm72Fek(NQG zWmM!plS9SBJ0wC_VT6jjaazYfA?KdMC>n4TTb}hsMT)lEYEUT5C)4~2&htX!^mRO= z9F~#@VTCy=T1L0j0L6>euNj3BS5cZ}f{MuL8X2H4(<9UT3eFkzXv?85a7LMSB@x1k zQK;DMAG!k+ex;?1q9Iq|n%ff<<6BSJ2#Tq{$TYu#b9Fnf*L4ELx_V`Q4?tKk9TnF_ zYdt`5`K5fkq|-LyDirN|pdup8UJn#@@5nU2g7fSiq#u7A6gH(KLRjH}idw-*OF(gT zR%1q?%vF5W?2d}l<9zHuk?ldI`4ybK@K-zW3l#U~lL%o&AS(29W@m$9?*(l}p~6+% z+hdFh--Y|lKymmgndVn;u8YTQpXPz0&TSGQtO!NLdj-|apm^WE8>3L=D$afGf{J-R zw`>E&)?s9tU%}an8xgx6f}&z9i4ayqpknav%-*2T$r`{Y)VPW|MV(R6;=TD6P)y1p z)BFm~UbK4bas?Eg&q#!@A{G^O<3xu+VK!qdqfqB69=_>>3WI|$-+7Wco{XXiSE1#uj|zLWj3iKe96_e}6<^tVQMdMyX0R73HY7q=aTpbc zo_E1Ecr!Qiir0KR-nkMAk#7`29^5`1BJDG(BA_PRwSdsY|FiMa7Jx$ zjbuWaaTW5m?NRY%(MLH@%SoNDk2rI6kV&#{)r$CW5@i3!k!Bs>&Y>SHA zMvL7*anONG^D92H_hMZ@tJ|PBGlxV7D{iAA$ah!;C`vzFU=%I63iZM^sIZ#ZRsj^g zH5~FXjSs?GAJwp$h3?K z!_1Nspx7BgB7_zBs94^4Xe-!@=l7m53Qeve_Mj#z;uRX60L9FwWSU>`iG4=-tvs;} z6q{d>2w_DLDil_gI)WnE<~^fm#Z}br-x3v1hIiWqiucpVG{1s#U3_ZYt{W(}I+F-t z#YdK+#UleytbJX{C|YwBk?WeHBGI%q{&VPs_hg!1!P$#(4V6ZM;$0bu5LSFgh0=-F zcy;&mP)d;emDGl-=-#j?Dm*%uJb}GPpHHUw6(8AW)Rco?o`RwzkVFV8 zhrO`7tHCI=xC&8XV^myzGwBE@@*k6F85RAMPJ4r*@Fj^5R@6mB@PX~#peS`TU=(e+ zit@#3sMw^QSPlw%S28W5qWy=a#-MQXCK1Al`l#47#C#?wJ{p-aigsMZmlzdPOewaV z42nm+$TYv=J^PGGZ<_Cz{g2H9=97fTetFV3D02R+OSC@c7c`}*iSG;4NQ7Jb}@Uy{4M-m~d zXod>q>6bf$!r-nyqv*g@>{U=i#p!`_dx2v16Ee-OC}Ar?E5;szy;xj8B7_whs5m)Z zw+|@FbXGD7U9Mu|3K1$M1j|1IMX4c~=2yIBD_#{gRsqF_o+Lt8(FPT(PxT!Oih{2j z8HFBKp=QDQa!96$(J)ZR{UX!+iZ^V<`{LI*pwOwm>F)suE3{Fu@zTUJP<;BbmkH75 zD(a1_hkKFLa7i*KM3rQkUs23fY%?iL28EV<(%%CRR_LN4yCNR{buuLTG!xR1s|X%i z7ZpucG|U7=)MGNuuP9;!dAr*K z6vlhVw2TUq&|Mp0FEkI62w{a0Dq7us=?;pnwhtLaC$7T4y%s8JjgG;+aCacnGAd51 z*4_e&4s%F^u%a6(w(0z?2a4MtiWr3x+u{kr~(qSAQqd{EFA?GwS_AJylQ? z$`$`T0AYm*DyA8Hxdw`fk>8n+&RoS0gG&6mWt`g@KRBZdHj`-?6|)~NKLv_2yGVqv z!W0#A-!2>gie&lviITrBjJS%E@5@o~sqQN^P^@W4ruh}G*n9EYdS*5#<~AV_!U}U# z$enJd3JQy5O&LWOu3~QBcU0t^R{Q{p@2kkPj0$fBJuxWiN0A6&g(WJ=j?e!HiV*n@ zjG`-7@wwJFR1_HA$p%G9Lo&^;C}8hJVAwwV*GZ*jBtlp*7!?uMoa8~#)vE`iFy<IIVsVa0G%bRV~5C@3<&STTxjT*dbppHMMt z{k?mj7*|22`4ybK7!bS26cj(}4*Yun!irI-=v+4ZCnyw?$1@?_xeBYI52zSprHFsA zo=GLs{EB?`Uc9|M<{Bv0Un3F1igBp0NI0wpijn{)M$v<-@EBHt3Ln*!udo*m%gHpq zB9E;&zxnh*Q0!StB7_waQBm2&JpmM#2Kz9Io?OM?Q*Tl6q;>3BP#BLU)BFm~UaU~* z7Y2&Hwj@GWF$EQ!ns>hqis8?L8HEW~QT(|G6&FXv;PawKF`4F9aQ33v8u?sMh(D7E zVa0S*c*kVC28G@9C`QqXtB@bhQtY2y3(u$yE@YZt@sxc=&5vx3J!;B45+SUZfeNo| zef;*LgF!r_=*?9mN4>(m2utmZ-)0%rolNs9ID7FsGXcNDxwJ2d5LV1W#pve;@Vm{U z(vLBUK3s+OhnJ|B+1POiTo(a1$uz%Wz(H0O8}T~S9nPpr_eq4X!W9)?N`lnkj0)L! ziBXtx74r9=qaw@p++a}n#gS=#g>SnGjOZWzWHKn^_K^r-g$FA3lx%thim|qL8HE{F zG1)y258=9EMmtcLIgn|7#iYO=7~!*D6+g)_olPQy72c>YX{(P7#P{V3M$wn6NL~C4 z4?%rn>}F7`dq<}E6*)&rF~T@0MG5xe;8zkMteB6A=Z&tK!(Lnp{LCo&aTQSkPw@~O za?cxqB0h{v^DADi`GOHC#eK6tu{n}N2rCw$!pzp{J}5kNYwwo)ePPa3Y~S@54E9*Exwii|#5jG{kxF9t5jMTN-;%^9GWV@am@ z6>{>QF~a9^?^$q0nGYus!ir#21SBf$14YVpBStZRtI(fw9}givB>ogA;_i}ZenocJ zdyE*+t`B}5b@wre5LPTl#WJnra8Nv7V9qEkxr+NPEJelwheS{m1(9ifh5S-hp*-R* z>IHjoU?qtVR)nJB_x8YKP&}(Wicwf`74?qX#f9vntzQoma}>!mzrsGX5F^$ogq{Y4 zml}x>R;)xtcVjCqU)?G7Oa;S1&RmiBtlrR78Tmg zQT1Uj9t1CD6oa{nNe%9x;<1yqB`B`1A=CVd0wY!x`!p;XKgoHsfkX%^B2Zyi-}$sA?8R=a zZH!_lS25W>3l*aWO~GHfb=N1;{EEFNS&GJb_Ip5K-jzfME22&yxsYMJy^J6F%#LBGKU{qZq+e*ss5i z3Wbnvv7k_KBh&nfX-{)7V$ts+e0DeYB@x1kO{h@cVvpZu(QNgUQHG9)__L4q7|wgQ7_} zndVnC?RO6s;-=~OI&el6)++gX0K$rGs94uNqZ25GCRZ>aqqqu-;wyLvrSVPiH3ApY z$TYuV%14%BddkYXpisU}B7_w?P%+T#H2!@NGrqwd$zMsMxr(zkEJb_oWB9$nxM^gX zUlG2XRbn>9>`(?p@JtdRtcXL!>cL;}-@A*GTQZ65!fNOxJcQ^mF)KmQvnQG6S3Eqzs$w%<=evQzryq$BR_sT` zfwa!MLGf!|98RfPV&h>8=#+Aagd^OIznUy;0kRmF}PSuO*`t@9*8SiyRoN!a-Z zR-n-Dy?{|n#*>{hBZp*o)l7WSU=LB4V9UrRNgx*Hxw~NQAH=85N0hLa>4KsCSi7Oyw#%yh%aD zCADYxdp-AtWSUQv6lR`nEJfB1eD^gK$WLe*@uooAN<@ZYdzOd&iwB%W5RE6Bs{kRuqWSU=* zw2P&<>Q`7F6p>aWLRfJT6_NQ_xu7`Pu`#1?;8C5)PRpeTnM8&)KjPaoO=tidb6$*B&y$D#e1b?rm=Sw1l z71vO)pzEOp@Q%G8&7f4tb@7$wWnPcBhMB>uz*2j3*#Ix9H0wae`5g@uMVJvkPxQS< z)vMjCC#?P;8e&83t#DA)S$5{q&&{Hcb~Hrh_o0S=zl*UtBU^a(t%{Yr|F%S* z#{TPtsF|uPsdE3Zx8Qw=%evCE>fVT|zI6s3SEI{!Hx92luF`vPJ489XXj08yiE3?? zkJ);f$Hnln$xRkC`#-%CH8>^00bbSlWH3z>ekp3R+Y9kbd7f5B0Z#rbYnmfD`8jVz zUD?_bb*=|go4|W>8&9Aavc48|O1npx8hux;ziHKeTsyUEjHv%Knp(s6q7MK3Vw_P` z)`rsCb|M=mn#FxF>db+Kc&q$oadr`0m`mK~d^K-IJrvRqukho|{g1&?{q&+ivc4LX zb9hFXoo1U?@pPMh-5}CfNK^m)Zq)itckvOYBwB7VpepZ3h|UgC$x@mp)5}og?zOjh-X--q zLmxfv?NmF!b<=DT9p`^&%xtfO8I7Jgq`;auI?xQkHIXaFHR*HBtP&vI&SaWOAn80W z|5-n#6CT9UGSdKnbWW$^{E%+Z>G<^9`PRx7AU|)=3_*ywh7~-X=Mw29v zWS+OR>{aWEkXAR`Cj#W?Svt-SIlrL;X6QdV*8(7Y(rJbuB(M)cinDDH@_Dkkp9Hd> z=k*#HtqkxWG;ZBD0*K8_I?fM?DL#QaGRZI~86XvNX@(#qbrM26b`Eb0x4kppdPYeg zJ6cuOM|2tl;8UjWQPU^z=R>@l2?-MDBcU_2aa|rnr*wusvQ2@8$lON)ehkFFD+*UN zsSkZ*V?!GIuRh{2h}B10Uf3K8eZ;&mO{=bt3^f^!+~DIYtf7zOwWLWk>mx6&>Y;hc zqi0{BkLYRB?Eln9V%!gngg#QNPZNdvh+PBL`YqgWBOFft$6aWSzE$AbY`_a@I_7Okx?c1P_EE-I+xP8Q6-VD6wKdBq= z2rkU))^xs_J~Da5G7S3A&U-d2m7EO?lC_U;4)0Q;?Mmn)VN+=8zx#+ycLl6p-TK9Z zLLWKnNb_XsBjzQnKf~vwp6U)a$PRO8q~Hxw&J%Bt>%!VKgFd3^NmFXpNA%2I=fKmi zTX5=T=3dM5?wM;2ZSb5=?>ICPt{b^Ebe#X8@o3gDd}x@rRGtZI61IV62(O7huE~>n zh8h6L>@+o50*T^zSxDC7_6Qk0a+Vk%IX&n&KV;k5XoS>iw0R#u2KJ*Ff{-1n5%M+M zdKN&s+Nj@`Ktg%m?y&E1bA;$wzOE0DUUqbxAEI`pA!bb6U2p;*na(sr5MsUsA*CPQ zd;*BYjQyV^kP4oc3MlVj?Z}A1=8FN+!i|peL#%VRLb1u8Tij%C-%}Hfj)9Xvn6iMWgo{E&_|LF(-4{a zh|R%8_?*djTkk0Jk+M@X_FsKumJ{oF>gty_5c){bd74&TAGw=20J-<|#7@vhzF($E zHR~gXD~GYn>K?k#N7mh>+5f4J#Elu14}E0KJ(?)oM;!H8>z5_+YX&F(md7+la`JQf zNUoMOu3Z1`Now$r4SGQ{WbGr)>LYN?9F@-Pg+6k#n5NdSkBpsQrwo0h>1Ue7?IQ~} zCt;1abRf71T$oLM(D`cmNQ)a6FzCqI?zdp6ru?Qsvi1?q;kk}KTO0aFmwFe;edFJK zWOuGM_K}R@{CMai#SLhlY<;BpO*}`o?iVeE8)Q3m8Yy^#+;I|bkoTjLOQDbSX-QLR z)<=rG%67m*<4U*tjV1Tmk34VO$eY;~J)c?^o56K6%#4onKQw0Sx{eQx_|_pqVNEow zXoldL1fIn;nG}*X6d)J##&(rJ3V2?T@W4F-HQq!Sf^JQwv#;itwGdBfmD%aem1D$(s>UlJeaVAUk)` z3_-~BHwf81U_nQ??df0X9U_6OHmk0W92mpuBPm)A$Kdmk*I6`3ppQg-QpPhi``xY8 z&_`P4&=8sXNW8-ue9n9eDQp6Lq~}u_`>#GSVIk{z>T>1W3Fsru3usz(ePoqB>z}9Y zcF{h!trkX2Wp8M5|~gFMvLht3*?4*hemg z>)(eyV%dadar?;GUoWsm1T-@>f(vtsCY`UQk7U%Zz@V8)imI?wH`~!5S^Eg*@X~vq z*$I7QzCKO;cOP-=-5>i%yC$37Kp!bJqIoj)kv1(9aX0&IQ)~%0$Q%nj;prr2@j1}*JI(5du=MuFZ9n=Ho|jWN#oo~xNe5|&~g5UMobLr z>G!hawLh#$mjId}xF&MtxF&TQM&kEJPVdv*FM%BA`APNJd{u-5kI32vkUA&nI6uVC z`yoF4K3}!*0LX|`njr|eU3Vb-dyU+R0T*FMT0Gl#O#<1&^BdUYa@NNRO5TP00Fqox z$N3?fPMt=`h^I@90n+pf%@Bm_Xn>G-jc7N3_zfHMN&?x!^P|Or^Q;eg#!L&`4v<0P z=r}*5Oe+Q<-vdTr)%`t%W(Yz&)DiOh<7NZ6?Hvs~TPcBD9bR1@xwDAXN4}5modSJi zNf-?h=p*4_3V5bIiP@J8ePn$E4UxHz*ql6yccxYyyA?wpIU7S`|J6svtztb-uV%JV zggz3wjiy!CN9LuOAoqTe{1@mW%Mxf(&H6~u*2QSvWuvnV`pEkIH2XjGk;yh4OrekT zJw_9S`$*sf*7_aq`uql*{1?v99LdSg=_BsCtWTDvB<;$FhwPFxnjvc+iAwjyHH+$Q z8v=dg_%)hZ!#=V*;m{7~BZqF&EN&k$J*c9JlV!9pnkutqjXP}R)eM|FX>LbeK9dS2x!`DZ{ zo&NS`8Yy^#)M$=3$Z3;?Z-zc%TTWAI)<+UdCdI--L*>$3QPiK$q9b^I(OWl~_1zCo zvEpaAZaUqd;ZR;W(2_%^3Cz!K3 zs^e|XKVp$8K-%ciaehcgY+Zg8AWu5e3_*ypHbRCCeEu0ASH@WNmq5Pr{PIuE zfb~Joybmkt0z`QV9p{I9yt)A)j}klA14u(Bnjr|u?}(80Cjuq_q|J#p(4j@ym& zh*l`8k6igW`X=-dZyg#W&_|9=s*Pvrz0&(m&_@CdXo$>x#NLbbJ0|zAiL0QG#2eGt zfAx`BF|6mQR;`oAp^sedP1CCDBZiqhksG2jZ2|O=bPJkPvp%xaEEw;`kF%^=K_A&L zgl7MzKBASqHw5~~l2J5KxR2bN&RV}4^*do7Sula-NKSrEA4wOn`pCjgb$Y==R(%@H zkhPCIwPdZCZ|+WG=pzT6Xlf1n$gTM!{h*IzyU{FeA36D45BrGOyF;zv!rbjm=d0-> z$(Fq_C@nd611wd!KMj(#k8lnzahr+~^bxZln)>fPA|4okePrI)>>1EUR;;3Vvh|U^ z!|@8}S+qP9Zjg7@(MZ7?WPKC7LFTDB)P+9sbR$ivSs$_WzTFQV8slCzUnaTN=JWiH zQMWbgn+Eb56FR{|BeH~!^FK7oOoH&C5tpU47S?2R8O;z}lfXW>CY}R7>H@@h$)fEN z$UUAPp1J*E-Q^RHzh488(V=vlA2PYaN`&~tjTHlAWdzL-gp3-1kcq*e?f@C|S}R2Y zx#ql@R1oOhK2BETU7WbPwtJyg;|zVoWgktgVIR@ayEO{>$hBiMi`z#kRR&^>FgL33 zg$r}i89HB0A4zqZh(S5>i_XGQ9Z01?vi1?q;rTlhBtjn!O@}s z`bf%6nkQ2qF}ml5yQ%JE{1|SKUiWFF;0?0C25*q9?sk%cK9cu@rqryDtUNJmAUrgp z!s=E^?zM;AxDSm7iSl^P?|B@f2G`B3b#$Enp^<%-^`lHJ`-WbyCMBC_hTxi{PQo>5 zTGBlZATG5dns4~?K~MZVF67hXS_m=Md7}c5`HFO$9}=q@g-?nN)8F?5$Xzv>Aqa_Z zK*+NVS*rl@aaUP)2_)8s3t8b*A0dr&W?u$K+I~9D4>8`k7a^4$J7@yr{c)Nh2$?<` zA^Dxu6ag~)@{BPO$m#`LNb@J(@F27b2Im08E{l%yLki4UpO0Mlnuqm2GlymfLX12S za{6cGJh<%*6m@f!K)(4`*GKBzX7v$upSQcAkF-{!K>~f`(W{rZE|vTGIY1v7(1M1@ z+(#VESg$^Q^+rD(`bZ}&8vCz4(q#|pc{+OB-dE5^hU(I^>iWn@3)bf&bpq-}Lmz3_ znI_e&k63!LzB7}UKgJRIh;I*?{h#_sc*CwkppW!0qlvln5ukt|iTp zocx?Vvfx&Kyb}Z}$4`Zatlu!2A!{EwIEmHa8YO9*hCZ@*3{93lVPWM7ak2H7Ws;qPl^&!R!H_7TqE?d_9h4}C<%ou>Y~ zkKAj#1^dYR06k0SBd>jEo=km2_ev=Ck$&4w&WAgF?Iko)@CNzB2XBzcBMiSl9~r-# zrqryDWc>f1dEudPVfuPs$-Oo(fcwxmQOf!w);#2r7hE^DTuaE+x@M=r znuL4O48b*#^T##Wk(gou=l1YRz1B-0Uju7#B@cB9@HX$VCc_n;6^Cv}&GS=MKUjq! z@+JGqdde~XDJ;5fD_V44Dn!sy6o{4(l+fwlTaQ@na@+S4Ey-7Iqb_O zf2nzXi+U7P`wnW@uPS%6ldrw# zW4=`GzskvwIM%~;^zd;{;db@AP%5svcG7+ltDQ`DTUt7z>d4F-?v$`#_a-6-$N4T2XrEEys1s-O z%a#?3_B59YOYW3FY=)+$TEUNhq_G^YJhqbx}zKOWQ=Ld49{43f2d7wrgFsfyL*LREVHuVk}xTrq~>W8)4ECt%(xL z?~T>Al_0sr_@0vRSwmMsL$?i+N)l)*cW>Onwdw3-brIUif(WS;ncK?7xvc+n_kDM6 ztcSL;FGec&Uu|U?yRFpiI@Au@O35~NT zE&3cxhqf~AfK>Q@YAgHQb1y+#>2h2uQn;;5aAvg?+p)9pR@?5ZRFLE{;Ix&KewO%o z%F?IL#qjvOm?jk>YgR+^g+cZ0Ul{f<-^x2<#>y9PUk z^$W{8aFH71N(EHYRut?uVveRHon^^VdeaW9Ec4N`ba=w_GZwW^LtK v^rS>MGYnsu(YvnVAvxI`*IX49F3DKo)jS}GM&Gt=z43-kU5K^Q)! literal 0 HcmV?d00001 diff --git a/logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/trial_1/flags_0.json b/logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/trial_1/flags_0.json new file mode 100644 index 00000000..bb190b3b --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/trial_1/flags_0.json @@ -0,0 +1,63 @@ +{ + "logtostderr": false, + "alsologtostderr": false, + "log_dir": "", + "v": 0, + "verbosity": 0, + "logger_levels": {}, + "stderrthreshold": "fatal", + "showprefixforinfo": true, + "run_with_pdb": false, + "pdb_post_mortem": false, + "pdb": false, + "run_with_profiling": false, + "profile_file": null, + "use_cprofile_for_profiling": true, + "only_check_args": false, + "op_conversion_fallback_to_while_loop": true, + "delta_threshold": 0.5, + "tt_check_filter": false, + "tt_single_core_summaries": false, + "runtime_oom_exit": true, + "hbm_oom_exit": true, + "test_srcdir": "", + "test_tmpdir": "/tmp/absl_testing", + "test_random_seed": 301, + "test_randomize_ordering_seed": "", + "xml_output_file": "", + "experimental_orbax_use_distributed_process_id": false, + "experimental_orbax_use_distributed_barrier": false, + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "workload": "ogbg", + "tuning_ruleset": "self", + "tuning_search_space": null, + "num_tuning_trials": 1, + "data_dir": "/data/ogbg", + "imagenet_v2_data_dir": null, + "librispeech_tokenizer_vocab_path": "", + "framework": "pytorch", + "torch_compile": true, + "experiment_dir": "/experiment_runs", + "experiment_name": "submissions_a100/ademamix_golden/study_0", + "save_checkpoints": false, + "save_intermediate_checkpoints": true, + "resume_last_run": null, + "append_timestamp": false, + "use_wandb": false, + "profile": false, + "max_global_steps": null, + "overwrite": true, + "hparam_start_index": null, + "hparam_end_index": null, + "rng_seed": 1297901242, + "set_pytorch_max_split_size": false, + "pytorch_eval_num_workers": 0, + "capture_jax_trace": false, + "skip_evals": false, + "?": false, + "help": false, + "helpshort": false, + "helpfull": false, + "helpxml": false, + "tfds_debug_list_dir": false +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/trial_1/measurements.csv b/logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/trial_1/measurements.csv new file mode 100644 index 00000000..054c40d1 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/trial_1/measurements.csv @@ -0,0 +1,153 @@ +global_step,grad_norm,loss,train/accuracy,train/loss,train/mean_average_precision,validation/accuracy,validation/loss,validation/mean_average_precision,validation/num_examples,test/accuracy,test/loss,test/mean_average_precision,test/num_examples,score,total_duration,accumulated_submission_time,accumulated_eval_time,accumulated_logging_time,preemption_count +0,0.4999997615814209,0.756860613822937,,,,,,,,,,,,,,,,, +1,,,0.5281778531601953,0.7542170812830864,0.0230337421923166,0.5284877849184416,0.7498691400731963,0.0271897867873256,43793.0,0.5292198959097536,0.7482351910609405,0.0283112904389152,43793.0,11.330527305603027,87.31072282791138,11.330527305603027,75.55893635749817,0.0,0.0 +1,0.4999997317790985,0.756700873374939,,,,,,,,,,,,,,,,, +2,0.4999997615814209,0.7577597498893738,,,,,,,,,,,,,,,,, +3,0.4999997317790985,0.7579421401023865,,,,,,,,,,,,,,,,, +4,0.4999997913837433,0.7551224231719971,,,,,,,,,,,,,,,,, +5,0.4999997615814209,0.752517819404602,,,,,,,,,,,,,,,,, +6,0.4999997615814209,0.7496925592422485,,,,,,,,,,,,,,,,, +7,0.4999997317790985,0.7457550168037415,,,,,,,,,,,,,,,,, +8,0.4999997019767761,0.7442112565040588,,,,,,,,,,,,,,,,, +9,0.4999997317790985,0.7389988303184509,,,,,,,,,,,,,,,,, +10,0.4999997019767761,0.739650547504425,,,,,,,,,,,,,,,,, +11,0.4999997019767761,0.7354312539100647,,,,,,,,,,,,,,,,, +12,0.4999997913837433,0.72995924949646,,,,,,,,,,,,,,,,, +13,0.4999997019767761,0.721907913684845,,,,,,,,,,,,,,,,, +14,0.4999997317790985,0.7181460857391357,,,,,,,,,,,,,,,,, +15,0.4999997615814209,0.712418258190155,,,,,,,,,,,,,,,,, +16,0.4999997317790985,0.7074881792068481,,,,,,,,,,,,,,,,, +17,0.4999997019767761,0.7017948627471924,,,,,,,,,,,,,,,,, +18,0.4999997317790985,0.6951101422309875,,,,,,,,,,,,,,,,, +19,0.4999997019767761,0.6871339678764343,,,,,,,,,,,,,,,,, +20,0.4999996721744537,0.6793647408485413,,,,,,,,,,,,,,,,, +21,0.4999996721744537,0.6717897057533264,,,,,,,,,,,,,,,,, +22,0.4999996423721313,0.6657581925392151,,,,,,,,,,,,,,,,, +23,0.4999996423721313,0.6592555642127991,,,,,,,,,,,,,,,,, +24,0.4999996125698089,0.6497958302497864,,,,,,,,,,,,,,,,, +25,0.4999996423721313,0.6440321803092957,,,,,,,,,,,,,,,,, +26,0.4999995827674866,0.6361322402954102,,,,,,,,,,,,,,,,, +27,0.4999996423721313,0.628525972366333,,,,,,,,,,,,,,,,, +28,0.4999996423721313,0.6233499646186829,,,,,,,,,,,,,,,,, +29,0.4999995529651642,0.6159453392028809,,,,,,,,,,,,,,,,, +30,0.4999995827674866,0.6064763069152832,,,,,,,,,,,,,,,,, +31,0.4999995529651642,0.5998926758766174,,,,,,,,,,,,,,,,, +32,0.4999995231628418,0.5900605320930481,,,,,,,,,,,,,,,,, +33,0.4999995231628418,0.584564745426178,,,,,,,,,,,,,,,,, +34,0.4999995231628418,0.5761562585830688,,,,,,,,,,,,,,,,, +35,0.499999463558197,0.5687248110771179,,,,,,,,,,,,,,,,, +36,0.499999463558197,0.5631676316261292,,,,,,,,,,,,,,,,, +37,0.499999463558197,0.5545494556427002,,,,,,,,,,,,,,,,, +38,0.4999994039535522,0.55038982629776,,,,,,,,,,,,,,,,, +39,0.4999994337558746,0.5430856347084045,,,,,,,,,,,,,,,,, +40,0.4999993741512298,0.5372253060340881,,,,,,,,,,,,,,,,, +41,0.4999993443489074,0.5283367037773132,,,,,,,,,,,,,,,,, +42,0.4999993145465851,0.5246280431747437,,,,,,,,,,,,,,,,, +43,0.4999992847442627,0.5167620182037354,,,,,,,,,,,,,,,,, +44,0.4999992847442627,0.5097603797912598,,,,,,,,,,,,,,,,, +45,0.4999992847442627,0.5056025981903076,,,,,,,,,,,,,,,,, +46,0.4999992251396179,0.4990477263927459,,,,,,,,,,,,,,,,, +47,0.4999992251396179,0.4944837391376495,,,,,,,,,,,,,,,,, +48,0.4999991953372955,0.490102469921112,,,,,,,,,,,,,,,,, +49,0.4999991953372955,0.4853548407554626,,,,,,,,,,,,,,,,, +50,0.4999991655349731,0.4824367165565491,,,,,,,,,,,,,,,,, +51,0.499999076128006,0.4776164591312408,,,,,,,,,,,,,,,,, +52,0.4999991655349731,0.4714734554290771,,,,,,,,,,,,,,,,, +53,0.4999990463256836,0.4685689508914947,,,,,,,,,,,,,,,,, +54,0.4999990165233612,0.4634661674499511,,,,,,,,,,,,,,,,, +55,0.4999990463256836,0.4603826999664306,,,,,,,,,,,,,,,,, +56,0.4999990165233612,0.4549782574176788,,,,,,,,,,,,,,,,, +57,0.4999989569187164,0.4537736475467682,,,,,,,,,,,,,,,,, +58,0.4999990165233612,0.4484002590179443,,,,,,,,,,,,,,,,, +59,0.4954701364040375,0.4451862573623657,,,,,,,,,,,,,,,,, +60,0.4924580454826355,0.4410783350467682,,,,,,,,,,,,,,,,, +61,0.4873073697090149,0.436396449804306,,,,,,,,,,,,,,,,, +62,0.4798724055290222,0.4345607757568359,,,,,,,,,,,,,,,,, +63,0.4756918847560882,0.4325987100601196,,,,,,,,,,,,,,,,, +64,0.4765647947788238,0.4289827048778534,,,,,,,,,,,,,,,,, +65,0.4833330810070038,0.424186497926712,,,,,,,,,,,,,,,,, +66,0.4726943969726562,0.4228460192680359,,,,,,,,,,,,,,,,, +67,0.4658952355384826,0.416833221912384,,,,,,,,,,,,,,,,, +68,0.4599687159061432,0.4168230593204498,,,,,,,,,,,,,,,,, +69,0.4526396095752716,0.4134599566459656,,,,,,,,,,,,,,,,, +70,0.4464377462863922,0.4122069478034973,,,,,,,,,,,,,,,,, +71,0.444900780916214,0.411267876625061,,,,,,,,,,,,,,,,, +72,0.4398261308670044,0.4082973599433899,,,,,,,,,,,,,,,,, +73,0.4295631051063537,0.4065581858158111,,,,,,,,,,,,,,,,, +74,0.429127961397171,0.4030865132808685,,,,,,,,,,,,,,,,, +75,0.4261284470558166,0.4013401567935943,,,,,,,,,,,,,,,,, +76,0.4237552583217621,0.3981367349624634,,,,,,,,,,,,,,,,, +77,0.4196576476097107,0.3956556618213653,,,,,,,,,,,,,,,,, +78,0.4185274541378021,0.3935122191905975,,,,,,,,,,,,,,,,, +79,0.4145438373088836,0.3936298191547394,,,,,,,,,,,,,,,,, +80,0.4158962965011596,0.3884983360767364,,,,,,,,,,,,,,,,, +81,0.4131359755992889,0.387380450963974,,,,,,,,,,,,,,,,, +82,0.414049744606018,0.3865808844566345,,,,,,,,,,,,,,,,, +83,0.4089271426200866,0.3839876651763916,,,,,,,,,,,,,,,,, +84,0.408670961856842,0.3829938173294067,,,,,,,,,,,,,,,,, +85,0.4051170647144317,0.3829579353332519,,,,,,,,,,,,,,,,, +86,0.4037006199359894,0.3814260065555572,,,,,,,,,,,,,,,,, +87,0.4013192653656006,0.3808557987213135,,,,,,,,,,,,,,,,, +88,0.4012000560760498,0.3756074607372284,,,,,,,,,,,,,,,,, +89,0.3947194218635559,0.3782773613929748,,,,,,,,,,,,,,,,, +90,0.3983874022960663,0.3731047213077545,,,,,,,,,,,,,,,,, +91,0.3993441462516784,0.3707645535469055,,,,,,,,,,,,,,,,, +92,0.3946015834808349,0.3716450929641723,,,,,,,,,,,,,,,,, +93,0.3923446536064148,0.3689739406108856,,,,,,,,,,,,,,,,, +94,0.392424613237381,0.3648853302001953,,,,,,,,,,,,,,,,, +95,0.3904910385608673,0.3677087426185608,,,,,,,,,,,,,,,,, +96,0.3862591087818146,0.3679948449134826,,,,,,,,,,,,,,,,, +97,0.3860142827033996,0.363320380449295,,,,,,,,,,,,,,,,, +98,0.3865923881530761,0.3596048653125763,,,,,,,,,,,,,,,,, +99,0.3818790316581726,0.3629131019115448,,,,,,,,,,,,,,,,, +100,0.3854906260967254,0.355971097946167,,,,,,,,,,,,,,,,, +500,0.033457800745964,0.0671956166625022,,,,,,,,,,,,,,,,, +1000,0.0306246094405651,0.0568154379725456,,,,,,,,,,,,,,,,, +1500,0.0174964535981416,0.0472845770418643,,,,,,,,,,,,,,,,, +1934,,,0.98727397833597,0.0466281751999426,0.1035014296223821,0.9846617349540008,0.0558138645153747,0.1095071933254516,43793.0,0.983643984067235,0.0591121106407387,0.106512755545851,43793.0,462.5499730110169,570.088809967041,462.5499730110169,105.69732356071472,0.0321097373962402,0.0 +2000,0.0344604477286338,0.0416964516043663,,,,,,,,,,,,,,,,, +2500,0.0239974036812782,0.0423150770366191,,,,,,,,,,,,,,,,, +3000,0.0409142896533012,0.0434757806360721,,,,,,,,,,,,,,,,, +3500,0.0222061108797788,0.0420215465128421,,,,,,,,,,,,,,,,, +3872,,,0.988466569277978,0.0401349372599171,0.1943110913960281,0.9854650596384348,0.0498583678504509,0.1658015498806869,43793.0,0.9845637651728084,0.0525903581492437,0.1699334133413196,43793.0,913.7683036327362,1052.229915857315,913.7683036327362,135.20834755897522,0.0673470497131347,0.0 +4000,0.0219005830585956,0.0391567051410675,,,,,,,,,,,,,,,,, +4500,0.0183785520493984,0.0407369658350944,,,,,,,,,,,,,,,,, +5000,0.0348568670451641,0.037563007324934,,,,,,,,,,,,,,,,, +5500,0.0195123236626386,0.0406799912452697,,,,,,,,,,,,,,,,, +5844,,,0.9888896233378502,0.037870113709421,0.2467587829208071,0.985922333644001,0.0477665181354911,0.1992099328688641,43793.0,0.9850217605722184,0.050569384355214,0.2002731426610653,43793.0,1364.8587460517883,1534.378156900406,1364.8587460517883,164.8463044166565,0.1001458168029785,0.0 +6000,0.024736113846302,0.0396488867700099,,,,,,,,,,,,,,,,, +6500,0.0183451455086469,0.0359333418309688,,,,,,,,,,,,,,,,, +7000,0.0148427253589034,0.0402457751333713,,,,,,,,,,,,,,,,, +7500,0.0189646109938621,0.0401570461690425,,,,,,,,,,,,,,,,, +7820,,,0.9895427617196844,0.0353866381484436,0.2761431838430722,0.9862570184354692,0.0464272905916431,0.2240062569055853,43793.0,0.9853462529448578,0.0492659202359423,0.2194470185799832,43793.0,1815.9603216648104,2016.2744965553284,1815.9603216648104,194.22807431221008,0.1358418464660644,0.0 +8000,0.0148676848039031,0.0351540856063365,,,,,,,,,,,,,,,,, +8500,0.0265447925776243,0.0358588509261608,,,,,,,,,,,,,,,,, +9000,0.0226705558598041,0.0407664477825164,,,,,,,,,,,,,,,,, +9500,0.0184488110244274,0.0375355631113052,,,,,,,,,,,,,,,,, +9799,,,0.9901857675683852,0.0330901716622962,0.3490473628829426,0.9865009790900146,0.0451063550228292,0.2486492353202348,43793.0,0.9857101223932164,0.0476459819537178,0.2409842598333483,43793.0,2267.014695167541,2498.28653550148,2267.014695167541,223.76578831672668,0.1679461002349853,0.0 +10000,0.0164722893387079,0.0345668941736221,,,,,,,,,,,,,,,,, +10500,0.0197831001132726,0.0361742079257965,,,,,,,,,,,,,,,,, +11000,0.0182838141918182,0.0359306335449218,,,,,,,,,,,,,,,,, +11500,0.0167634300887584,0.0306401345878839,,,,,,,,,,,,,,,,, +11778,,,0.9903584234755128,0.0318247757834728,0.3719740073263724,0.986685268835711,0.0448715023003744,0.2582817253603897,43793.0,0.9858520483238844,0.0476167353940529,0.2583378703119949,43793.0,2718.106857538224,2980.2949731349945,2718.106857538224,253.27358293533325,0.1994554996490478,0.0 +12000,0.015896301716566,0.0346976816654205,,,,,,,,,,,,,,,,, +12500,0.0168644692748785,0.0319809205830097,,,,,,,,,,,,,,,,, +13000,0.0163647402077913,0.0322085097432136,,,,,,,,,,,,,,,,, +13500,0.0162138994783163,0.0314379446208477,,,,,,,,,,,,,,,,, +13753,,,0.9909471630031864,0.0300438163663444,0.4012493489435839,0.9867646270852344,0.0445130994643013,0.2676612510009205,43793.0,0.9860992605532668,0.0469874062846655,0.2613127550481985,43793.0,3169.12438416481,3462.3594183921814,3169.12438416481,282.9000051021576,0.2322547435760498,0.0 +14000,0.0175456143915653,0.0326298139989376,,,,,,,,,,,,,,,,, +14500,0.0180857628583908,0.0315818227827549,,,,,,,,,,,,,,,,, +15000,0.0207067485898733,0.0332999490201473,,,,,,,,,,,,,,,,, +15500,0.016754575073719,0.0302563179284334,,,,,,,,,,,,,,,,, +15735,,,0.9914554525443318,0.0278195623943076,0.4653121079507286,0.986941407226756,0.0447404013558691,0.2762624088256628,43793.0,0.98617275038977,0.0475611442283314,0.2682699677631099,43793.0,3620.244943380356,3944.166640281677,3620.244943380356,312.170524597168,0.267136812210083,0.0 +16000,0.0183285381644964,0.0314843244850635,,,,,,,,,,,,,,,,, +16500,0.0170705541968345,0.0278500784188509,,,,,,,,,,,,,,,,, +17000,0.0183625742793083,0.029556406661868,,,,,,,,,,,,,,,,, +17500,0.0158081836998462,0.0270337872207164,,,,,,,,,,,,,,,,, +17711,,,0.992003039266106,0.0264366338691659,0.4929768841281881,0.9868512919766576,0.0447637071333527,0.2784861725845394,43793.0,0.9860162949498796,0.0475549454974946,0.2634218412426757,43793.0,4071.263598918915,4425.883248329163,4071.263598918915,341.4536712169647,0.3021092414855957,0.0 +18000,0.0210767071694135,0.0288593862205743,,,,,,,,,,,,,,,,, +18500,0.0175930447876453,0.0279048401862382,,,,,,,,,,,,,,,,, +19000,0.0201180968433618,0.0298432130366563,,,,,,,,,,,,,,,,, +19500,0.0175765864551067,0.0273488536477088,,,,,,,,,,,,,,,,, +19711,,,0.9921952268836712,0.0253662223338853,0.5251250965686298,0.9868525097503076,0.0450054827721561,0.2820029958838089,43793.0,0.985951859735066,0.0479657265723245,0.2710933780442979,43793.0,4522.396051645279,4907.878260135651,4522.396051645279,370.93720149993896,0.3350741863250732,0.0 +19711,,,,,,,,,,,,,,4522.396051645279,,,,,0.0 diff --git a/logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/trial_1/meta_data_0.json b/logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/trial_1/meta_data_0.json new file mode 100644 index 00000000..2b429d7f --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/ogbg_pytorch/trial_1/meta_data_0.json @@ -0,0 +1,70 @@ +{ + "workload.activation_fn_name": "relu", + "workload.eval_batch_size": 32768, + "workload.eval_period_time_sec": 452, + "workload.latent_dim": 256, + "workload.max_allowed_runtime_sec": 11303, + "workload.num_eval_train_examples": 43793, + "workload.num_message_passing_steps": 5, + "workload.num_test_examples": 43793, + "workload.num_train_examples": 350343, + "workload.num_validation_examples": 43793, + "workload.step_hint": 52000, + "workload.target_metric_name": "mean_average_precision", + "workload.test_target_value": 0.268729, + "workload.train_mean": 0.268729, + "workload.train_stddev": 0.268729, + "workload.validation_target_value": 0.28098, + "cpu.util.avg_percent_since_last": 12.9, + "cpu.freq.current": 2200.1639999999993, + "mem.total": 359053524992, + "mem.available": 349600968704, + "mem.used": 6296735744, + "mem.percent_used": 2.6, + "mem.read_bytes_since_boot": 48898170029056, + "mem.write_bytes_since_boot": 131330520064, + "net.bytes_sent_since_boot": 31237504, + "net.bytes_recv_since_boot": 31240466, + "gpu.count": 4, + "gpu.0.compute.util": 0.0, + "gpu.0.mem.util": 0.0277587890625, + "gpu.0.mem.total": 40960.0, + "gpu.0.mem.used": 1137.0, + "gpu.0.mem.free": 39190.0, + "gpu.0.temp.current": 35.0, + "gpu.1.compute.util": 0.0, + "gpu.1.mem.util": 0.0277587890625, + "gpu.1.mem.total": 40960.0, + "gpu.1.mem.used": 1137.0, + "gpu.1.mem.free": 39190.0, + "gpu.1.temp.current": 38.0, + "gpu.2.compute.util": 0.0, + "gpu.2.mem.util": 0.0277587890625, + "gpu.2.mem.total": 40960.0, + "gpu.2.mem.used": 1137.0, + "gpu.2.mem.free": 39190.0, + "gpu.2.temp.current": 35.0, + "gpu.3.compute.util": 0.0, + "gpu.3.mem.util": 0.0277587890625, + "gpu.3.mem.total": 40960.0, + "gpu.3.mem.used": 1137.0, + "gpu.3.mem.free": 39190.0, + "gpu.3.temp.current": 38.0, + "gpu.avg.compute.util": 0.0, + "gpu.avg.mem.util": 0.0277587890625, + "gpu.avg.mem.total": 40960.0, + "gpu.avg.mem.used": 1137.0, + "gpu.avg.mem.free": 39190.0, + "gpu.avg.temp.current": 36.5, + "os_platform": "Linux-6.1.0-44-cloud-amd64-x86_64-with-glibc2.31", + "python_version": "3.11.10", + "python_compiler": "GCC 9.4.0", + "git_branch": "main", + "git_commit_hash": "b21be29be0a1573fb4f78f849aea019cdb520862", + "cpu_model_name": "Intel(R) Xeon(R) CPU @ 2.20GHz", + "cpu_count": 24, + "gpu_model_name": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_driver": "550.90.12", + "rng_seed": 1297901242 +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_0/wmt_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_0/wmt_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..24e51842 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_0/wmt_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,34 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/accuracy,test/bleu,test/loss,test/num_examples,total_duration,train/accuracy,train/bleu,train/loss,validation/accuracy,validation/bleu,validation/loss,validation/num_examples +440.9525320529938,0.0,76.32085061073303,1,0,76.32085061073303,0.0007088489919237,0.0,12.044632792981233,3003,517.9636764526367,0.0006042089422923,0.0,12.05177956633758,0.0004835649898947,0.0,12.05185847044674,3000 +633.0395290851593,0.034538984298706,718.3431355953217,4156,0,718.3431355953217,0.5523676718377781,19.208453271442007,2.5487264685375632,3003,1354.6345691680908,0.5489787078830985,25.51018287710872,2.59488128618976,0.5528263753704232,20.77749199731472,2.5477869152273374,3000 +858.4411356449127,0.0700490474700927,1360.511590719223,8323,0,1360.511590719223,0.6170937191331125,23.751160079640105,1.966464942478648,3003,2224.6266939640045,0.599668911976253,29.484997493987493,2.102941617193744,0.610816976850876,24.84724125678546,2.0018062004810853,3000 +1041.5366611480713,0.1067402362823486,2002.6904723644257,12493,0,2002.6904723644257,0.6414037534135146,25.611929074622587,1.7672175498227878,3003,3052.3292145729065,0.6162815929830172,29.869366276144724,1.9544963967153577,0.6350696209594425,26.66075888044792,1.8175107872190053,3000 +1236.902271747589,0.1411404609680175,2644.968838453293,16668,0,2644.968838453293,0.6559642089361455,26.4104526056849,1.6560678853640114,3003,3892.406261205673,0.6284328084950568,30.84180050089463,1.8549527559730867,0.6470967501952859,27.63565652379737,1.721576414117618,3000 +1447.8440327644348,0.1755435466766357,3287.2435598373413,20845,0,3287.2435598373413,0.6640055778281332,27.026689310797952,1.5898763871942363,3003,4748.056781768799,0.6389054066416613,31.288025568108456,1.77046037359345,0.6537922654399821,27.8906182266799,1.6578037625075943,3000 +1646.4197652339935,0.211613655090332,3929.4358291625977,25019,0,3929.4358291625977,0.6709546220440415,27.231862769804216,1.5374762143396663,3003,5591.25407743454,0.6479801763383853,32.448172318625886,1.6988456606926756,0.6600042156947837,28.196767541148606,1.617126570036329,3000 +1865.9192254543304,0.2468822002410888,4571.648406744003,29188,0,4571.648406744003,0.6765092092266574,27.98602990649897,1.500034679855906,3003,6455.38191485405,0.6450799086757991,32.059144285066935,1.7099643264840183,0.6662409641541952,28.76029220781709,1.57774227458432,3000 +2119.0626318454742,0.2829687595367431,5213.796993970871,33360,0,5213.796993970871,0.6807390622276451,28.493779819136066,1.468950725553425,3003,7353.10406255722,0.6554331141606909,32.579644058317534,1.6377410206992142,0.6679768384768943,29.096854810739405,1.5531466209346445,3000 +2321.044037103653,0.3195197582244873,5855.98249578476,37533,0,5855.98249578476,0.6851664633083493,28.74394052805401,1.4453388276393004,3003,8199.691377401352,0.6648527895520678,33.55789683123363,1.578003637300951,0.6717337664753071,29.151142586857883,1.5327072819927838,3000 +2556.7816529273987,0.3569211959838867,6498.172658920288,41706,0,6498.172658920288,0.686456336064145,28.718436773021423,1.4227539800127824,3003,9080.049595355988,0.6605701557997221,32.8269526340634,1.5903092314492704,0.673891210276376,29.618207209392207,1.5142536399114703,3000 +2790.200603723526,0.3936951160430908,7140.401864767075,45881,0,7140.401864767075,0.6897333100923828,29.01833424750774,1.4008395793387949,3003,9958.129945755005,0.6687904705841615,33.26489292961415,1.54301550578852,0.6759122639520899,29.25567465177173,1.4998456892970948,3000 +3011.202365875244,0.4645464420318603,7782.6305565834045,50054,0,7782.6305565834045,0.6921503689500901,29.1076151397941,1.391680829120911,3003,10823.819381952286,0.6735101619014813,33.436885790865766,1.5062322382018605,0.6768669948295744,29.795995027663388,1.4853608255012336,3000 +3223.396757602692,0.5028431415557861,8424.923278331757,54228,0,8424.923278331757,0.695775957236651,29.462941877924937,1.3669638966358724,3003,11680.741777420044,0.6695961450206517,33.85869854742274,1.5323578763194126,0.6803139452703624,29.93200702476283,1.4672586204758775,3000 +3447.381803750992,0.5438311100006104,9067.07328248024,58404,0,9067.07328248024,0.6966591133577363,29.3762478734997,1.3583626568764162,3003,12549.31753706932,0.675087410563364,33.75690313913292,1.495353600331118,0.6813926671708969,29.651606972963968,1.4579864167834249,3000 +3680.249090194702,0.583716869354248,9709.232525110245,62578,0,9709.232525110245,0.6990645517401661,29.629351525062223,1.3470577501307304,3003,13426.77290415764,0.6781173649443489,34.9030195664067,1.473514575179848,0.6826573756060061,30.034928923864545,1.4489792439027414,3000 +3895.9259502887726,0.6218104362487793,10351.489212274551,66750,0,10351.489212274551,0.6992969612457149,29.915111229994693,1.3391317689268492,3003,14287.140409946442,0.6760110346722221,33.795533798759124,1.49228739082657,0.6852983843969697,30.305291905739445,1.4414615131864452,3000 +4098.024610996246,0.6599886417388916,10993.722551584244,70924,0,10993.722551584244,0.7018418453314741,29.9305108094369,1.3263530590901167,3003,15133.90243411064,0.687737344478783,34.64019747009979,1.409397041675695,0.6861043260467942,30.260549972098406,1.4298347393398718,3000 +4312.904577493668,0.703080415725708,11635.910709619522,75108,0,11635.910709619522,0.7033989890186508,30.17727112013422,1.3169482816222184,3003,15993.4167406559,0.6873241369844211,35.10220106458447,1.4139003986228351,0.686401904502114,30.20928326719492,1.4203690902778638,3000 +4551.652169466019,0.7410016059875488,12278.122849941254,79291,0,12278.122849941254,0.70403811515891,30.196260633743304,1.3051324371041777,3003,16876.808160305023,0.6846116111662502,34.67072321627014,1.4252815662183282,0.6875674201187834,30.36090824007107,1.4130921733766475,3000 +4775.149796485901,0.7820522785186768,12920.284372329712,83476,0,12920.284372329712,0.7054093312416477,30.197715887844023,1.298196102783104,3003,17744.912157297134,0.6931581866075142,35.55717010011654,1.3781471677295625,0.6898364558405972,30.42063895411213,1.4083055386789995,3000 +5006.733048439026,0.8224046230316162,13562.501360416412,87661,0,13562.501360416412,0.7070361977804892,30.371250564178208,1.2947403731625124,3003,18621.147614240646,0.6964027301420904,35.634105035055384,1.3526700890771528,0.6899728459659521,30.732287328799785,1.4038192916082877,3000 +5228.371054410934,0.8618402481079102,14204.70498752594,91847,0,14204.70498752594,0.7080820405554588,30.630104490928616,1.2840292908604962,3003,19487.43827891349,0.6967524734334921,35.62226473493535,1.3594542629511726,0.6922542807900708,30.529455857116726,1.3970977188751534,3000 +5434.9788410663605,0.9011752605438232,14846.929310798643,96032,0,14846.929310798643,0.7093486723606995,30.37390140044731,1.2777466990587416,3003,20338.708476305008,0.699206924732668,36.02363252187802,1.3320539563117972,0.6921674870739358,30.503563924140668,1.3897272274987291,3000 +5650.112952947617,0.941009759902954,15489.126465082169,100218,0,15489.126465082169,0.7110568822264831,30.83824125061178,1.2748164328046017,3003,21198.4762544632,0.7010576275827448,36.29287865890685,1.32520942908647,0.6933825990998251,30.703973921269075,1.387492773648188,3000 +5878.489018201828,0.9821929931640624,16131.295519828796,104404,0,16131.295519828796,0.7112544303061995,30.82440532248409,1.2717339201673348,3003,22071.462913513184,0.7013492118124871,36.5867640368117,1.3243644019182672,0.693580984736705,30.80043463228064,1.3883863808260282,3000 +6101.3889355659485,1.0220787525177002,16773.50749373436,108587,0,16773.50749373436,0.710789611295102,30.62603422876235,1.2702854460809947,3003,22939.01176381111,0.7014952301481698,36.552775460332526,1.3235886016891816,0.6941885407496498,30.74154060181069,1.3850052037482483,3000 +6314.789815425873,1.0644431114196775,17415.770446300507,112773,0,17415.770446300507,0.711905176921736,30.748248276620576,1.2670732017314508,3003,23797.128108739853,0.700723999496746,36.62687539942267,1.3307948732429,0.6941637425450398,30.74221672807699,1.3828152123036292,3000 +6519.404279708862,1.1050832271575928,18058.02095270157,116958,0,18058.02095270157,0.7116030445645227,30.769477198529625,1.2678339797222706,3003,24646.43512058258,0.7042175616501498,36.602375940877536,1.3043353163171238,0.6942009398519547,30.71857660929905,1.383598196240592,3000 +6729.55979180336,1.1453070640563965,18700.251742362976,121141,0,18700.251742362976,0.7117192493172971,30.775598462168684,1.267700162686654,3003,25501.25637102127,0.7014666438395252,36.53351868484419,1.3255141456942303,0.6942257380565647,30.655805237988723,1.3836071080953738,3000 +6940.523397922516,1.1880042552947998,19342.42072749138,125320,0,19342.42072749138,0.7117657312184068,30.767489851609625,1.2675796002556503,3003,26356.818597316746,0.7062242562929062,36.344055673993815,1.2931648669908469,0.6943249308750047,30.692571533837658,1.3830785963596235,3000 +7170.2508709430695,1.231386423110962,19984.64667034149,129505,0,19984.64667034149,0.7121840683283946,30.779927367324674,1.2676883606414502,3003,27231.190740823746,0.7083323734709392,36.57114632263505,1.285520877726969,0.6938289667828049,30.72696537138564,1.3840151160555976,3000 +7393.343722343445,1.2717046737670898,20626.889546871185,133688,0,20626.889546871185,0.7111266050781477,30.65709660704041,1.2693124128464355,3003,28098.93967318535,0.706277744003854,36.788203593365424,1.294006327353437,0.6942381371588697,30.880696709612252,1.3840120162800214,3000 diff --git a/logs/self_tuning/ademamix_golden/study_0/wmt_pytorch/trial_1/events.out.tfevents.1789292648.b0019b52843f.38.0.v2 b/logs/self_tuning/ademamix_golden/study_0/wmt_pytorch/trial_1/events.out.tfevents.1789292648.b0019b52843f.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..c57ce99798233a3002426a4b3d59ecae4c8c02c0 GIT binary patch literal 119180 zcmbTEBnlxEA&E!`AzQW( zB}tY@!uy)rGxzzOIg@z)dOx4%xqk0_=ggeB&dl6X9~sF%4{r45bM@Q1@~6@b>hY=I%MStD~ovb5~yA7?MU zdn(>KztcV)$}*R)|Dtb~Uum7%DYR9Pb@%l49-vUSMq?Ru*(c;pUAukx^4Icv-~O)EQqWY8ckyy?vh(otn)zSA_n$xeK)_bhk_7$V0rg_l zY~)K@_LKadcT~_)P;_zkoa*3i=k4P>>%WL!|1m^NxU{3b(eH@e^B0ek&u@HS+pq0c zs?n*lg0?~fpI`f=n}egHua|@4{Qq(;IjZ)8oU;jDiwu6}Own&KNxqdTpVL91y@G-y zhuj}6a{A%^Xs~=kGsrmHo|Q4x-P!lQ?GSkUhGb{={=FUcmhX$@ExYg;-4(hjG@0Yz z?&jp+_6!a8Y5L@KoJJZg2p2N&p?#}!%DC1{h4#cAawCeZ# z9avC!WrF1^vBkA zZ3Yo{CNLtZ+&kIEPhw4eW0W=GC&}M;;3K*!=qNPewt~4|5~%%B$9%@xZoi{uPHr)6T--cdAn{i%B(D54#Jd+KzmnHl$0v%ddIH@$v` zG!83zF5f4V59y-NSwU%*m$UQCS-*C_ouj9(htGe5AlRHNFGH`czmqOr9aka$Z9>6+ zbEIIFpIBsqvzq_C6%*mOJ~5TAX3z6K=y~4v*@DxsKTb{OBdWbWKK_VZ1`7R=A0&#J z!fVyTyObhi`dkN4#Es*t*$Tnc%HWmyc~E5A@e$$*nSL;;$Bm~h0fo)r?-E5Vp(6ek zr6~6pdlD2$Bl&8!;wL>8;V(W$gCcnxA0e*jhl)m)l|4X_WLP(nyXwjc6`_48MZFqJ z3_-EEH($+G2*$!}aM=n_xR~=1;tD@hSeVan0mY(DO(criLdEJ2z3{x4LsI>kopWtTSKDRtU~+U7gl9 zK~dj;j}TX+p+d&&+80nvX{;?#$O#o$PmFOamW6(A42oQ3zM8EN98m#Z9!v*CQw=^s zT#<*0O(~X2plDjxK%%HCRNSzp6uUb%9S4d!jreM|LNFGWVvf!Sg>oxCLR=w(dvj6C zqc)%jY1mhykQXXsf{n2Hh11)4pipkfSF;s@u^8TT(@{{gSK}kZ6-u}rVetkIplDoS zDN!g06|rl2V)g4UX_=tdQe((})og`eroD18${!Sa<@pG4g&J;0)`eqdL1FyXTCyFA zLdDqkJ+L~ayS+Lnj+XJ&Y=vMf=3ThC7!>i}`3P}EcifJ7F>C99V(SAtiK3oR5&Fdt zt2d>c@&U#8XMA-v73V)RJPeB6ulWdZ1=ZpI)R^)H6c27rmniBB6=j_%g@IRpZ%~-# z^3~N;EDTe^v6%dvj}TXk!o4}UvzrVk9FO}-6b*!m+4=fdJ?Vzc22gxG!&kEvf;p=B z`Oht3EDA325#kDa+>U*(qnmkM{tD@zS*tg_2MaovMq~xxM5*gW};zzPg%PWl+SA z-z8Br7AkTVbi?Z0?wwzQLeY`0uBM`Qi!14%Xg-6F5LX1_c6e_xIRc7P!{Q~1CPIbw z%&u6i{VE6hs3dE?x|)hs*WcqDl`@Hs5Lblbb~tzKRu2@L2cD29nhF&Y`gXx;!?06N zU@Ynk;fp7%a4Q8W`O7OCoBb*=7A>_8DX zh_7ZVzR`*^vD0RO;@}8ALR=At+p*~O9qi0^b-5-{G#4tI-gLt1R*`YZpeQxutJw;{ zSll{pr4I_dK753@A_2E!^zvJ+U@V?#-J7V?ZNE;JS9PGwdS5xt(hMX5D;(GEC;)*of zjy9)sHi5!bqeP-;B~;{RcED=4g;{t+<>>I$Y=z)@5uqtJ3=}4Ye1y0n6SqTmcgZ?X zIJf>PQ78))b}O{7deJQF!7vuJH2G?_qJloV)l8#oL1Cc7M~Ex3aXYjWr;Gwc+eWh6 zxjQ+ng^G23+heuP$FOoxWVGa~tEsq=(8&)J?(O&pam9Vyj>ZWIrl5E(+fbrV5h^|_ zYhv}A)1w!FBD)@6T}{O{+lywPuxiFfh%54NJ2ci_#Pj0y$5s+W8==BrkOo#SZf+k1 ziYwpvYPLdfy>Okd^av=v)l&X%1H=`DxE*cPPOgWs*z-wK5~3`zc=-{2#}6?IXuyK<=_ zD9-JlC{bt#75!^cidW5tIDz8kQNEh35S-odyDPA#U3i+05LeVgMHh=!13|HH_cVz@ zQ>eIUse)tS(RMiA;Y>KdSF;sg=&|Ua+p#{3MbR-nLR_JQ3Pt&M%AjZ;Hd~@-FH|J; zYmEx+eT7S4EWCH{)zwt=Iqhr=it@dDgt($9D%zc$UKbQw{T4|ST0+HS17%dKJX(JS zC`PU2tJ#Xr^jHkmG-wVA-%WgkxS}O0V&Vhwdhx`2l|<1&s91Zv6)K{)WLtvb^J2c5 zttg`vNB8f%0E*75_y}=@3My{aIZ+H_G1YaWMA1>GSkRy)Djv44{|OY=XYti+#YbA< z8htte6ms+V2yulPDjbb}CW7L%!*+>6Tc|KO)&dm;?c(v?VCU(4bu|?MF>CSe!DTN# zLR_JViU-N7FM?vPW1K|MNvP;_usJG}6`Zj%zcPcbW-A0o)Vb|>8(=Knc=HkBijJsQ zur>=7SA0e*jj*3&ev#9BEX)nW-H#)XZP5{C;NcH+m(+HR~Vq8snyCca9(Vlm?Kei6)M{JHNmks zxIU{EC}ueF)zwr4<~5W7#n>5qgt)>86}9x%^Z~`WF;69mZbHT6H;qwY7*LA$27gZE ztJ#Wo^jLg&z6j551t&g2Tw#g|#Z3ixx4GYlB8j5AP|>bGrD${NXj>SITQ+=kH5Fr5 z9_$KZp=ifPh%5S`;$U~(w=foNR%H@}u26B{O(PtONizB|pcrh+SF;tx^jK`UK71u8 zG^g?r;tDfV*nV4{1Bx2MWp;3Pa`c3XEPYBbyPe7dP-u$5{14{5i(8*$71j#Gc!=6j^L}=iXvJucZQJ(D6ZM?5#owrsIa$m zO$EjI;msuq1EE6MyCEuahwr)uicvOvHCyqTR$P5-V+xABQ}_sR#VAzt(ef$*h1u|S z5{02qap54P2zgvk0g5wY`D(VJfL6Q@PJIIkxhZ^vxWXD0A4a|G0*X3=bR>!%LWR{h zN};)Z^9@j>j^L}SsW{h89p3|~H;#`GSBytRucz(zgW_=Co)Se*q2hE_13aScjjp2w ziYXR+HCvHSk40ig4&J97JBp7GS4=`hft}NLQ23b*kSL6Vio*BxQK4_@RR)UN!})5q z;yJBIt3R_n98m>h`3P}^Jt{5+?i>mV4a?yYg|Sfaco?PFxXj%M6xXfz>S`)Bx!ueH zg}p5wA+B&j#o=2&{6G;oV!TArOQ=xqOex;KFTYtdSl-)ZMt~xAJRc#ha7V?5 zYkTmBx^M0wQJ4x9ZweLhyeJ4hyciT;NAT5b#en8-QDNRN<~1lj+VBzLidm>o*Q&Ju z#v<2bzC_VmsHp8;4;7|;Uo-%PpBZ1xR#;d+MTPt5sotQNK8%kLSNNc!G;Y~LP!t%f zkSO{H74a$5SSa;qgm1Ih_vWkFih=_uJA*JVM@#4pm5UMBT+DZWw$Ss8jCmP z(;mTC9PPqavlW?EiKvK?u}KDndJjHAT;Y$3p$mP@K(S6ML82HS9E&C&g1OK0;iv5*5u~xoW^zY^#|eQJ4u83AHJO#@u*WP`p&&tJ#W& zgLa@o_wG1+e$l-NA0e&?L`CI^b0(nBugsPx%!LZgebf>4@OTEUySvoB`Cm0#ab)r~ zRA@V?sKQvxsLw};E7qXGb+Ka}C~{>UNw#BwC#|SO^s*@9N?l)j4>4xYF(#C<I|5DtbqLN(9AoS=IkGKwJ@y zimpZvc7meKtB#V8;X=jjp_Jm#hM7M=5mv%ivlWd#E<(kc4$*HxG50ecA+Cr(#q!AB zA3>4(N?)QFAyhPpqZC_~O_m2m#5=y4tw`4OKt+=`yDou3xtxy>S45)1+Re`x6hm_Q zNE9Q53Rf*kG5_-#eEV_wBfgrgC@FPE#j)7(-k?~L&qs(WcA(-&UdCxqoV{x)QH&BQ z&Mu@By32Hi!&uaQ%2%@$J?BqHMZe~Qc7S4j0Usf*h(g7fwm+|dqRBmLiNZ>#*mpt> zuX-&u)twEBu}}GGwqkTSrOZsQrXe7<4?9 z4vL_=d^KC)x6ld4;>=P{T~K5^;UmNqF{n6jX`&`5GSa6@6k~*ntuoXRbvZBE78Fme z^3`mG;WIl_95fx@0~9rL_y}>uUQ{Hg_3H@=)$_g*g|$%eWe7DEu0{6;f@1a+zM8G5 zEOSQ1z#un#Z*a#gK0;g(hl*A^4X%R1@}!?cF;=LU(uf+1{Pleog5t$FzM8FgvDgX~ z5zXUJ@$oVrA+Feu3fY!RAAut3{91{^MyRm#qQ*jfZNVu}T)e_pvlSWR$D*Q5rCd#nr9a#h}nE{!7pia zdU0N&m?%`Z+EGW;=|yfQK(V8kuVyP8@~QJ;^}|bbL80-Pj}TWJM}^g6%Sbpcp66ea zC?*LNvX`l`=%Y5k0u%?{^3`mG*;Q*CiziKo%Yx#586P38NJd4?TdG!|=y>P8L@`;Y zn0230Y&mT39u%RE`D(V}r~!3$FaP)*->2R5l8+EqoI=IYZhdjZy6(;kiDHUSA$O5d z%+`x;3X1EG`D(VJpW7rH3$Jn8TwyHQzv3gr6)C7_VLS{K!4FC#3Ok{q&QMBWywM!* zN69?rtJ#X>`qZ`C(_f$48VR@8VA{P^4w?)oexU&jV0# z)pR0$o8@#4A0e*DK*jaq8~9a|Q5V}w6iz~g?N{o&_}t7FXWBd0_-eMo&tniO2KwIo z1Y>dH4j&<|$V5fEMUCphH}-5w_Z{ZWi%*P?d7V3d7e5wMN3k}3a?Be$ofOl zayBIR8})jNcT4>N0C{^{njs3IznJIl>0T-= zXG7jvWZ~@l=8B;fK%SOMGejY2wm2g1E~yv;WL5d09vmc!@f+QiGYk--dT8D}fXu9M zm@j8TUJOe{$lhC%t^j1Byfi};BC`M?9eSp^z}Iz?THJN#ASr59eZ=_5>U#LeP>p3z zYQa@4t(`PT;QAG!S=hSgaEyP3j|_L<&b3?M304%j#3$tFb@K zrAhzvDdUV*BlX~$yPbzi6UDz}T<${cExYtrHQ>N*W-ZO(4s5~4jBQk@{d9EfQE$l5 zoyf`{_fXOPt++;VP)BN^84JRS;Frb ze-6{f%g|zlH?!ar+c{fWU&RNFM_BK}pf2@FZw>mrAzp?j_sG|qCk?9Z7mWp*x3Q1q z<$k|YJGS@~Cf{PIH1*G)G!7h~id*^expv;b-+4y@+DFNM50K_n>zl@D^Xm7857)-% zeD4QS-IjIINYSY-Zy8Q?t%|%8;0xE!L!~Ky_f=z+mo4YR^nd;PpatBl8Or!6dSpaL zoC(4joH+wW_|H0n`EvI3-_0`tr~jT`j&6f}^0t9ALv)`+1>ruiA9l?WAbZL})^U(P z#&3tshqOh=y^gDQ0OaHkX*nC>oxTSl`bt?V0WzfSMt&1SA^jo|Qf_?l6hPcJK8fNW zm5iUPT1U4-$f@*z41jdpCM{<}cGQnS$mjl3v;h*lTbdyX$vc3MwfkR;0?79VofA1o zDdRVWb@r+tWQ1|pOMnb~AuVS^&TQX;kO%s@NdVdSMw%fC@jH!>%zRTFfXp7Z;1UPP zXZ$Fv#ek*=i9G6N3Xq2O(sDNB_M!-c)G@S72Z*<;G(!}kb`>FSyB9_QL_hXwE(f{K z_!W<3dJBY@znjz!AR`l`1_)7~;_?n4p>EQ0HpJp&076>0<-`Ky{8w0dWAycteYea66wCfH`l@aGFpAhR0!FX!VDohuR+h=VCVJvnoBTr z-{I-fkUw?bep6J?WfdEp4?Xj_E7G*8y6^BdGKkypJn#l2Mc))pB86I)GEitTWoQ_5 z-}w)uN&nP+n`MT!gzmdEPnsy+eLwu#2KQF?^_x`Tz)mcX=5PnLp!-%!`i2>;gET`S z<7EjegWN;*HB)g9O?F!y2HkgFnKa`syKmj=?rbN6i;X@gv^ykp_e zeV4YB=2ffv&hy=X-S^_)WV{(C+fEuOI@Q&yIS6`qwH+1np!2QPxXh_T1(|^Ds>Rs!srFP9=`cLmH%@Exu*|NA# zcFkoKb*zi}}dUs}$FYzy&1$d2xw<^YjjE6osv%xa2| z)qX<`0pwUor&SyznDK_}fwgsUL^jquqX&>~U!>)1h`H}HghXmS9u1JBpVACbNIw;X z1XxW;0Ek)lIT0LW1>=>*=9RS&629JjKS0Wjq~&bL@By|6nb!A1JV12&OEW|vN}33f zsdeZJK(6SgALJnAj5p~jYse$S()oB_fXp|Qma`!(hfYDreUrzR0pe#S%@Bp;bw)^0 z;!=D*WV%Y}90w_8y!vD^lsdHL9NQ)b5Vv4yIU91`X&6FU?bg@=kgRZNhA1RKA0g}B zjcf-a;-x<^hlAuX-c>m{L=H#fXw(ZwfcW;2ma`#EGlw9gv|?c&fVd2jW{5)kOc3I5 zRLLA#1C{v$n(h3xjJW1dRrkH?0@ZiTunKJqjkC^jX^=?wU8+-ojhAA^)&&r9WDPTf z?7ZfDD#6a{VB=YNbaA6_z%ooC?X;pRKdNGBV)Z)amX^>R7 zQ#^?jnp)mL;h`g@51{)V7AH;mr|$bz#}@0L`*uAnO%(6G<@Ghu`0jMOr*L5RJ}%AS z4s1d9-Ob=RW+b1I9RwMcXIL5J9x{1JJ^!ey7~K@QZ|e)vjKA!@)jmblhwj_ysx(X3 zeYdos9>LmlI2;A1Si{@W`YP^yn_cXK-M3YTpjps;2R)DmRk!;VY~I?ZC5g~|FUXUo z{<-_s(e=mfd%NGouh4y`6iD-`)qO`rj)x5DBOm%zTNh3w4I)i~Y)s0>BpbvE4 zdp=83{;vBzzBuhJO#daCrzEpx4&#-qVJ~Xn^xx^d%~?3Yw|9}2v#0;m@q=;tPk6nq z2JDlyJ){|;`(zZgPr79UUj;~mZXY{x(|@M3@J%3#gA}AmgvTR*TsD%Hvmt92Q?CL| zT%Ym{Ag=wT8KMxip*XlVXZ1e`kieZiOgTu3n-DT{9n~VmdA!&FklJz5ayG=SSw9?+ zua}zC1W2C*X@)4IbTmSq?Y;3HAS1{7jpQH+Glh^Pr>POSvUt#CfT%f0%h`~#-HZ|9 zQDY3Qhl*XL8KRIhTZ9xiE_eozbcc+o93*D85Rz*|E$mtud|D2Wg)^k(Y{<3Uy%BQF zs`Vupk)b})3{gn%RD?|Mn-BpIxh0BoI7qm!5Ypw1ES~I(9hBt(a(<<>oDC`U>Wh%g zE|c;B(rJS`g=s%edBHvS1fJSwH-Y)JYmLxhBP zk_!Y#m)chRCWu11dm-eY;~{r|EJ_NC@UUl$84n_)N+^-l{5iyu0%(DYjFBqGIE z6y|R`AhWz!Q3sZU_4g7%+#xPlQq--ujUfRZ?ad&h?m=bTvU8|QG8$b_ZuSHmoI-UpL2=`6Rs-!Mz+Vrz`zXaR5;yEo+kpTSXTHA^2Q|j zSM@^%|K5~S>EE`=r(7nIs=KZbY~a+nb6UWknm?UQ#Qu3<@ia057ZwdH`dOO&-p+v& zx<<&)y+;IAYh|IfN(tva4c%tLVD78`l*kmF`;7c??u%I6Jph)QZC(*Ef48>SlG&jM zt_cff8GqvD(ZUssYl2%7>YDKGV!=l^)(6ce>e<%>Ew?VXUplzYS_S(heHjrVx?i$a z;C}h3qxBptb-m}xMg6+F^eB*FQHcM9=WTB30Q@m^(L$o0ZPA<60WAZ(17yI`Zv_z| zYDo@6%igapPr-6K`h+rP$qr&zl;tUl%WG#B7?-~BL_OPb_-IG8wAOeZ1Qy*yB1F^@ z6@->`m)0!<%Z-?dE}Z2;FvF6%jj0<)^Q?4)Djqs zmWmgz--Bh?#XkKxOL8c~;?SDv6EZ5^b)6mim(oW55y~K!k`|#)qTj zOjvshu+;3D;lf#>A{mxFo4=q%Vek{=gXK$KB1F{EFA^>4 zM_T%TWyF#O3pq<@6vMJ>VG&wdOz&L);}W@wsApRy|4>ECYZtQzV9^UELPRYcqtJ4H zmvuTc4o;f7XE@8-=qmg45gRm6^>S{Y0d&}HyAnwveY)2U>UM8sp9`LlQojc~g>269 zhF4&77SU$acj(hM_a;*Q)TftBp}NjEi8#E^=+l)Z5|LH(>6O~Z?AtZZ6Ap2;sYDQWhzt7k z#>=i^h~M@MysntKF+<3`w66#C#|THaTsso_^kK7zkiYEHC%e~u3w`>mxkQ+-Pk-e- z9J}j~v0tvj={9dE5l}^+?qeT{Nq4QXjzgc`b|sNi-9BBgfx~-W-v)j9%ymTUpZoNh z-x9G;Pdy~J5&HCzp+sP{`t*cz`Z)Iu3z|0%=DsG8M5gH6mm7_9pY4f?Ptd0a?IvRW zu1{}e_@NeD6VxKdr*rdYU<~7$aC=uN9_yMGO87a|X)#1S`cN-?-!3P;xZ0T!(Tp?5h;Wh}$e-k<6RZ)AU-04M+EV?;gMGH^}{9OF?&wkBZd zovq1waXIWZ3jfs4kYb{qZBcvQ2Q8WX6~2OH z#wQ{~)RK1qEjQ1W>;+56Hr4N(CI2wPvgOiiv>aWz4S!2!8bj2xEs0~ApryEGO+BzU z#Sn+{PAo19mC z#r6KpxTe7DYv(o^4q?qA?z0{$~9` z!P2=25h7~wJB^mqLfdSxoVPL_!dVi}GAwe&)SUciR6QTCB-;}8Y|HMN)cuCcxkFOG za(^ljB5H9xi-6q*Z`fFndb+wxJC8ka#1`|g3| z^hqK_)G{g^EiM+LE*e4$e}F>W8jWStW#yhz zH_-b}?otG+g?{IWB#|{ohW7*9H|;~mzJipym)R*~qgisL27Z5H+|tQCV9n9xCXw=| zHHZA?j<`{$M)WL&g+hb-L|j#Cjv7C%W7Ju{cNGw|;b|37B=XzFXSn7#+^_x#SaUcP z5K;fM=Fpp%TLx>6h!P@Fe9dt>|qnEL?t()hP06kyG9v(^vk0sfb34!wAdQLyIFR3yTLYmTp*eQ~AG>B-w!aJtoK zLIhN?=BRt>5GHN8kgE%8jtQ-Zr0T9Y1RGec+;9V|IWDRZv4383cwW1PYmQ<2L$|=1 z<8TKeuv%-5en+S;v-O?g`~>E{;oXQ#(YbF9HTP|=f4U8zb1F9dq;PT)u9P2}S5%uhA!s%tL@mf7>t4B@PF9!w?A)@=m=n`HN zngsTo4wl8&za?>&;HwPFsaC~!-cFr21^-OTlzT)y+v4ur5-ktqo(%%a!>2@us7386 zT80d*htCEMx(>R`SytR&Sgwat*Xr{FHn_pKtej2Mvn^v!Xrtxp;*z#tv71kXh+692 zKuhSgGHyTs)Z-eEe%+q(ArT8wxa(5QB7X1Fu9A9d7mnZ7k7W?+*Kv{geN2BMD?c%$L>0&BJd5IZv93P0af(r4NhOiq@qVDQ=m^zwjq+L+ouaQa6;WR zs?ev;pG?I5xlfqVwD<>6RgqfXSbuL%{_ z!*RddkDquOt_gvii4f8KGVc}cmv*zyrh=u#vL&lHOF|LDvdEn3(;wx<;Df=`)kHnp za=MlpTHbz+s|%KG8;KB6i+vGVt`$$74;G8K%MqL;`{T1GVw!4KM}4<_o_ zmhwBj@EGgeOiLLoS4I*cqL!AQ(UQMyk`Y*x4fp177QYIHrK1IPZ?d)BP<+mDtS?c| zwiI|$@9@muG`%NSLM(_7QHx9kT0RdRIsz;QN4+WFET6tJEaS_lcRM=tX^HpKFW3_G zY|9fp>N$t9f536DxHu9aqL#w%Xt|`*c_~=#_UitPvlK3?DSX#{(mTrXY`lFSoG%sT zL_OP**OYo*o3d(JG*~o75Fz3gnZdBG-BfuW-?g_e@UhtaYb}_)qGnb1)8}5HUUZqI zS$P^dY}Y=d#&TW?hp`}Cl~5g(vWFCIpu z{Hae5l%ehp%iDPzgXa3AH4#@;pYDGx1ASjdm#IS3?uk`Ik;q9Ih1jRR+dgz3^y%{) ziKu_-({1P7Z47<-s_8_ec%NQm))kqbPfvLYhxp9dL=bm~3;OiXIw=@(gs7PoU@JIkifryq#O&YB~~!^&E8LBiq3tbYU142y!-7F(5Jf}A!7cnPq&Vnc@wS)W{>NP z;O5as0gP*cOc`~5qODatu1V7hhK^pwuy*(y4zr&+_j z?dag{;N{)9qEH7cTB;6*I7`%4hUJGBW!Z68b`e-!YZ3KqOC2NXFRR?kJog>O#i$z* zB5H}Mhn6>=U0#DF*L&YN&JwzvVcD2Z-B0(pnY<4y`xg=QY|G&f)cQ&O<3m@lXao=; zqL$DGXh|Bb;SH9{KJRllOW-bs<)#ufCwE!1p*C1D7Zdeti&snP@8x!GXpL+1u`7uX zQA?l_TE4q~vj@x7Ge(7+r80(LX*rs@-%x8r5dK*$w+y15Z5a?p{r$i&1KAodE@{_^ z5K+sb#%MXBvcEGlK@)OYsl@zRb8O#J<(lKFK6L|q+m)aMSS{q{6Gp$X zsdpenv6!7gHky_ub#TpLHZ&y;)*RbE5-ERLbEJNz)*KCXDPM(!LSzLISJj&1TsP{y zl$vWw211l?4Q1*6rlUyY8pGGvP4Dtg9tdj=6?r1+pVk~}kL~^jYmO#LM5Oqdqp?dj zY&*^`&d0yXRHFqE#2w;-HOFI_lNgdQV?Y}?=abtoL&&|f<>O1-ORcU4=)$vs6PiTG zU#>YuKHN4M)*POli7?@s1|Gc-p$lt{%zi}dpVu6TN{vul=GIsT)*NRC6M@xQbF>WKjcbm~J}Dbv?rSiT z$P}IXwo`N8gkAH~Va*|9L&W^un&Zap3-92XF!Ys4S8g6HjbmIB)c4%NV|~@*N%)Pl ziV~upeNDJleS@Poi%HEJZpScWWre@MAYKc94&2|uK55Kt6>+cIm`V7hDELcb$_r{ zY>Uxg(Hl$Dvn|gbP=EEXqNAK5j7#ifB1F_Oz9m|I-n`fiEDOfebLA{qi403OhsQWB zGYvf>z~VlMsApSF1XJVkvb|YDu;@4wA)=OH%4k_rcc%-lD2c|Vj05w#eoq9sOND;O+#R|1FfyqWEDD|VlQ>J{1%~BBqdPb*>wBN* z2jj9r#@YMg$~kw z^xNqV&GnIoL>%2*lg~bsZlA-b+LuP-yY>mstB4|zEj|=u2YTw-_IA*x+ZPd0|J0`s zyfZu-`t;N9iAeE2J>oRgryo1{H3bgwRpmqwcZdu6^lK`|abNA$zNG{qZDe$%$EGxd z+)IH2x~u}_b1va*9d-A#o^s&1by*udsz4orYPeX#}+`{zF0KdUYF>B{BBs?et!cOn9- z)u-F19>=-w*!JW`F!z1cB{D_lz9MSw>*2h5BJ}AmjEI=O>(f(i?$d>9g017>T6=!o z?+DIhToc|DP`|KcH5h7|S)<%o#x?o!v z7mcrHMsb$Bdkl+pIn@s;eOBxR;OwcMIuDhlGhn6eGa;B z1xuX=HJvz1&O?Sp+m(8jmUClA5?ErM5%p|~+D)preX+a20~B0@whIbG1w_u4aj z*S_+i#azyk`IKSlb>SMGFN?Y>;`hXw-XQAPmYb@+I4&lGC*%3D=N=IvYRT+|mWE04 zlVDutyx6dYv!uLWSZ3u?@4V^%xQ_3MWfc?kY|Hg)3(#`w?d#DnE|z6Ph^Qq+7cClh zOYxMtc`toxYqq-=+n2>CsO{@r*E!Hy*Ykz>^(o|(?h!^OE;>jKK(<*S@flU>tzm6 zF@{w{k;t8jMcAirEbk%%efqNAMASd^>3y3U7(kyMVn#%Y_vw>2c1Py$`F1N{>bW$O z2;vTLL7#qn#4!xb|nI;=+gtjTVtPoT4~LE=+l>Y5=qtV(*+wi!ppi1^y%7j zh}b{(={+2~qIk`W*_WVCZ?c#OtX7}yy74m3ef@72Erhvm;R+&CbnY|O$GI|N|#Qa^Ku6IaoAzTw?n=Zc3&7%ngjBCPz_tgFL$9)Y?!U>w`^BSIb!&ze9Ff8tasCN>Kea!GXw*@VUdbXv@Vh^-5 zyE*kGSYq1|A)=NTLmXqdulaRgT%zqOe{h!YcMOY*JGFkg`SrRVj7#I`L_OPLtTh)c z8@njt?|c2dh!9arcu%yHZf(#4EM04vHQxJcEg1BXVfj!=y&rqVWU?1nLKKO5wxyqg zKU!X9`G&x_J_ zh?cYiJ@Gf>Kjng?8zE}(GeJwUBUiCcpJa2#h_igEU|7x#puP)T*zX+vzW0Y6QO~wa zUP%3EtUmPy4VK?C8?c8%?{j)S6@G+-=KY&0(QOr2J{k(Qz=f z<}fz9+Z7fHON@!Qs@5DM)KamJUSim98bl50S49+w)Sp8ARkKEcDK%it(cglI`lmI= z(D3G$V9lXAl86*vb6n`Ii_BZ6%wNJG-pqyw;tp}anq#Z_Q4BfuGjceDluTlVkbBAb z5w+&X?r`iNtU1h`h>*WrbDTPukpgRur89^y;hICy@FRBMCCT;f!Rgk+n+T|4%`v#G zHm*7JWVdvIHAm)rBB{D-4#5V7*E~8E)*Orch}b``IeP1xqByGSgTt`qn7fJytk#;N z?t`~DkLwn`SOjz5$Mr;}=-juGn)_TIKGuOX$AvH==I_=VyR2*VfosBwWj<%PdGx{> z#x=o7FAI~DnkZJR71LPRZUL(wvF=Xm@s<7({< z3bDV|PoWVEOV1zFJKcQ;Ssw?>6+NP!ZRwM{0WDt@ray*btey!GB5G+m94#vj$1B3P ztm|;E4QC15!LYdBrQQ|sSf_0Yma=X{J=>Chm>L(C@+tVd)Y^y$5w+AEiI&Iq9q?OE zKH=?jIZNd(hUL?F>VEp9tX?m`q7Y5gvn^_V)bGM78XPWwaT&ay2obeZjzY_J?eD8# zT*^mHH{&d&dl;71U$gOiS-EOO7qIBr67_7$>KfELJo-K(XM$zkR3b#wQaTzfC&tG- zg~nm>$jSbkB`3DZK0S6i^=zP4bH^djVGo=@B#HFtBQ&XZ(O1kbwt$pz_Ush0IUDII zhtCGi2UyO9KHbQLNcmHr-awoBwd(G~q|eY?FYqAZs_N6*-8hY7|MdQkW)L;Tw~8nd zndb8jpABf~y~=<-y~$!C>Yw`b6N+~fp-=z1oQM?f)9qFDka?n++#@)|7px_MxIN`(AnpPpKf>H~fH_9!Au*ry-a zU5o0j2ReAb>82G&1XR(dm;3j?r0B!8?7*g*Mu#tLb31IebC7V_RE(e zM2P5q$+yP+a4_SWGFIB&g&5cO<}kI7oJ=uK1}0+#V( zh!9cBeH*mY?$dT7Sl+Z+c#N}T9bs5LJfxmg7oJI43KkU&qMmKJT^NFvr^Ds(IsW#} zM2M&*Ydl&EdQ|KHOK|gZ8Js08iDB8bkb1wO^@Ro5U}>pN)Uz#}il}}tuD6mcj7y77 zM2M&*%@!>(i%;O1Lw{|ZT+Whsl3}^zL0JxKso{Ow0ii@a+v1o?^@F-ORz+aRi6lZq zEs2xR;_h*N8H~%b>qFjhmbfzvOXi^SIGr^No;U=?#r!@|&$h@s*@DNIR{x>;U^$XU zgos+=rl93~-~RZWf}rIaWa56UpCZpOEYrNG=Nxa&)x_To$OjSiY|Dm*5okGB%Rdq< zJwk~PQA?yfTE-tv#c^r<@lF%Y5`2+id1pnBO5He zWbaBhLevsG6)m5?`)fes;L*f?B4>%otg=sEb&I;6ZdN-3|61)6RU%2GPe1EIz2ox2 z>}hN0)9+}pQ^@A5u^iQ>zaKxf9`xytx)3RU>eJIayW)xT=>GB9&|F_PAmXa((}(Xo zg;68zhT1{YIg=`)NaT_pZ?OY4ZMfqG^ywi3iKu_-)BF1OzX5%E>JTDQyie~pp6b){ z>V@Fnyj^8Q1aXJBpij?vauD}b?(`;jk8$>RW(c{L*4L!^bcO3}*Fc|s%#H~8%RW7^ z?wuUy)7!cbVZuIrXgu|X{K~kISKxGu^B@AM=+n)cnPHzEQDo=GolkA8glj?@W1^mY zO^7}hhS!8B*V%=zU()&$A)@Z>yh1 z)Uz$J@zk2*TKFd;u;^tHA)=Of({PMCUd(?CmNmA|0ys-?Hp3FHL)|yZ8aws?j7vu+ zqMmJ;VjGX+BD3Ge9xM~wi4akXyDM4_A37fjmL==7wsMxd9EN2=DfOJ=N5SdKU~%6} z)Uz#Lj&DNCtr1V~IsT~~M2M)xemYumAM{QHOSe3ac+Qe@pJACPPu=Tmw7k*{EL~m` z^=!+6L)7|d+siUs@Av*dgos+K-O-YhU@-$M1A~sAjnfUjoa<&s8q81Aew8Za<$M=zRr+mM~SyJ*CmP_5~{%z2c-C*h9Le#S@ z9(qxDjGY;4G6l|;4W2}ZsKs;^TJ{_t90B9^hAQ)__bqA^j= zww#dNh?ecM($2%Us0<)NL@k;=Xi1I@)q*DIvdYsQ`+ltjKm6Y!X@Z!4OG|kx^`cAj zHOfC>wXj-?ND^6ds6|rWdo0`BYCojx@4`+Y8_gq;iui0GXp!6wSaZBEAX5Ic=7>H| ztvRmG-=_l$g_|ZsTvcn14)dsQ%OpH#D+^IO22>G6B3BPC#WlyF)1I4Q&EYwOi2A2B z$HKA!>tW52WJN@ZuQ^_pQEQGl4t4PL!Mo#$Anp(stT`Or@8Sx+?Ym${Nq3$3h~hx@!)>1`gJ(sRnC~qy9wfpVu6fwWi^kWB9H){;=jqT1^C2 zYt1pwqZu9;cN<3Rg}JX?2$3l|_jyrsU$oEb|NnIl$v>GVc8``X2`6IyZp{&`Wqc2= z35Pyd4Cdz1@M6X_;oxiP{-BCskS`qT8@>_s>}x_qEVbsCZLFsc(E4bweAr6Vvn{PMsJ_Gbs~h%EQ=*9wQA@pf zXqo!xE@ZBGq1(t$~L_OQ`ZP;eCypXRL1LN}H8WAFD zDO`+}TCTyj!E&kLkyy_1=qJOn_YL)KRbgXA7qE0|Mbxt`$D+k>7r>Cx#t*TFN*pTYe_e|Caf)2ZUKanKTrw^({tu6KEdVGTv z*Q4wdvN?%SU9pMKy55%o`f`um<%cc4#yf0u|9@6*+4>m$=<$9`Kl zT{}M^g1AFm(5D}8J%AxAd>Va$kohl}A>>}t8cn_FE&J9;(>9;b%7GheHkr?OwqZ|&L8K# zp67}Kpih6JO2qtKpYEQtvpZZ9#vQrXf}2N^*D|gNzKf}M?T!1m*uk;B=oC@Uz9u;I z-jCM=jgo`YV81*(PlSl>mq*loxv}t@5m;cR3M%FMc8GDF& zwq;F!s<$1wN*;d$G3gKyB5KKAftDz*KHtDHBq_*>vxIJ8Sn8)x&jwa5*2d=?H_j0C zY|E6x`8X~Qw&j$IzSxWcOU_mzMAUL&C0aUdF0BO?&z-9CI7{UYh9x(V>IYl=_=eAH%43Oo zwk7{1)u&%y7;qgd0}_Z3QA_G7w4@$hFcU0SzB{euETz#5%Z4G;F{Woe9RH5qxH`_# zdbVZBRcanC@Y~Bx1_YVrLsjNx8yKA)B+{4GMTaT_buz0`%!;ZxShg>eB;bs6M^H z&12)Cxvu$uh^wkkzb#A6`#Wk3@P(-2r&UCe$ZgH2KD~YO;Cs-gUn?M@{;5x&{Nz$P z^yyh8M5K71UU-1&)18v*Z-qmAPZ<%!9pZvMz4YQfJQkM5xZ!tr&Qvl($i0*>o9ffm zKfm>XKK*8`IOzfYmwmdu)|&w6(_IvaFkzpbpV1k+>pP2%tb)_6aT6k-iatGZt~(|T z&$LU1K0Uq_kyPD2U9f?zHKsR*K7FMc5&P#pT~Emm`}BlqUC%+E?$dz?tX7{cThUyLryG$eI`^$ygL9wvJh@!#(+!E3zw6W0XAV0G*MwlDi}~CwL}G>rOyyK zUl^B8FK4N8mc-)>i=GK(@p<1_4UVx9r9?g3;%G^|JM5&}!vZW`~%3fz* z2g{K?ak`u(?i9mv>Ir2j=+JXBSkev?^=!+zTpFi<@QEoka4_Nk3B|=0k>o=e!rECp; zAE~If@_5b?e4b&kilo+06J8eXf^k_ih^S{5*o_l}ygK_!(fCv$__=lk7{lvBS zcgRPr4qC!l$}cl4PBGN;+S;{7<9C01Z6xa1mg7CC^F=Y&0KY{M7D0rFS{7|Y%cIKu zbD;^69}$?$S@N$`x#n2#iE7Lfx82Tx)xyqkM3TswBcXy?bL3nZ}+5I>NYmTs;M3`{RvFC;% zt`s)L8{CJ}O)HiNsAA1=;N^Twy4q!d7OXkm93+yeyXFvVV6o5U$*|@~OC)0dyyobA zX(g^XZtqahgf+*}(?npk)*Ma0Q!84t@mJ@-Jl->%$P}IXj#6{qtV{WmV9l}q3K8>n zYmSYnbF$(3R>LFh5;u<)Ut?SodS9lVi6#uKZ2`ynm^nl}`dLx;CGZPT&$g_l*BtGV4=#b@ zwZ2Se=|+fJriY|CH z#|Oc3W*reCYH`|(mJ46&orPnp@95a3oF(Nk!}4`w1|DNBxnJ?`4YZ#~)Uz$_ZK-Di zG9O)WjlSKH2obeR*@71LdGGN%JQ|H&b>J)s&lr~MM$~V}ll{A)<%}{>&$g^^pvI-S z!VFxax7Q#-L@ndP(V{kBNdp*{8{IWcI7`e+hQe-g|PpLlr z=s;y07x(@|h^S@sRh^yyKa z)Zb6~)>p0%^yyc(vs1|C?A9Ub*}&jnb6I#c5VeO$`BR@hLza5%A-`-<9yHfy;)%Gb z`gDUlY8_%UzTrlQQcSEOibQI@pq>rHYKD74pML2y5%o`f`p9Pwy1}!71?fbjc%OdX zgzD43KAf-s4)K*&h#>9|7xd|&4PtRF4lSKq3?X(mnIYs}dLKgd>2LZgt%N@P(tRT2 zFZ*=2@}$<#r$;>{!i0VLlKXwJyFM4Pp&U-P9)(0e6@B`S2g@-j_gvHM(5Hu$5J}bT z(*+w?I^xj==+n2C5wU;n(~Wws$3A_e{TWN>)4eK*z-smB7Ds#If$?Pf-or5WDalTi zPFiAfpWkC@?pv}lWj^%jEftBFzw6T%8->h=YeM&-VcWTRH0UkknlR9l>N~zxhCGL3 zeUvp(&%P#{&!nzzI; zTrd14tKTi6o^83JaTF~NW^BU0rL`xQ2obe(-HDduMj@_XarklIJ7>xEt1Wyk81;a9 z=grDA-T;oVUUKo$dbVYKU+Ua9ZF%<*j7$H9M2M)RV-#BU-kX4b6VOXHU+Lhlwcv#n z49meN>Y2?Dvt{_3p^v?YdbVXo0(I`^fk(^z&^XlT?CQZ; zLW62o^{)L*FX~g~&GdI%g$}!sA(14~r>nUn;3c@kXpI4+EH-7Qkj>fJUJdYm`WxM- zPSB@M97v@6sZVdamFm-nbnDRtn(M+LL|j#U`ow`$pMKldHVdMzTU8N7BIiy1i1*V! z3~y5p`t&suh^T++(;dHL*+QT0VoyYh_vuMtRG_qoV*lKy>l=q*pKcOU z`~~{-LE%JTwfgi>0}DJb3dSEe1#@4WC?ZpI?o-%}b6=f?l^38-f3cT{`MW;-W%kdt za7{S2V5KKFkH-Ih9u>U9)1e{NL#-%WS_e)T^A$ur`bJhChflU(0I85X-5)N}m3*M0C^`?%3WJ==0O^Do;DFefk4iB1F`p zum>&vy)It@%amQ3`#4KrIK$HL67?*%Q+{oH&e3W=QO~xVx_SUDKOGhEna%v8M2M(G zb}w2obIkXFC0TY-3TLU@#;`2DPxZDrZI`cxW6ZcdQO~x>H#vf^2SUztdLPRa) zu{bXG78N^#rFHK0=bR;fH^VYKlj?12-TKl5EZ3hC^=yl(IyEl2YhK&{%i$s-MAY&j z4lUn956lLOL!iQE&T>DNVF|LOdNs%1vG2e#X(Lh3w(L4d{my1-lg0R(p&t=Mh^VD_ zA6i}>)R_Pl%?LC3L%-HfSqB&vn}yWh2%5KDz8+Ydv_x(A35 zQOm3SXlWXl+ZmdmGM%}WoaJqNm1_>~!PKW$bnPRb!)n30Cy^ww<`}9;eeZF|%8~OR zWqu!a3fX9mSxv1uQlCyWhBe1tb0XzWYmSD|)S9FA#m75gp%6Zdh^uPNVViIa+oh|K zGdn<3+?Xn&NaV?fWw_>Wd~vQTtT}Wi5>fxO=5V&WW(;eNMN^4L@ioU78AD`_8M6xi z4Aw3;B8WT01#6D_r}kio-sIIY;XHAk#S9_$(&%#P2};;nyTP#L(4I$x{NA=oSx z)*N+~5@EtMM~$K}xKdD@(a8)>w@)jHfGXA;!xn^KQuGKTSy*#ST1O;Rcg-Q#z|#|Q zPQsd_O(+rj=QYP;-$-0@>{q{40Ba8OZA4(T)*LFUN8^EUx=udM;~jPrnPPLFUylPg z_uUDpa|zZQx%-Kjzgu(ook_%RamaYb+Hmt|+F`~u;ZXH)|)(V<1C5C7#3&UGdMh74DZ;3CDV+k zXIti`Qtu=*NNQ6XEcb>HA)=PtcpT&9*4CYgos*l4x*)Jz=M%s**|qqFlUK8&9E3hrTX-7PDR={&BnFIZ-3UFo^+hEtVm~A(!8iO$H3Z0@LTXQ z-b6jyqPdHDS7iOimH6Jv>V-szsO5YDT4qK+!ZnAZ;)lzer96XS3EE0M7i==}tO|_F z?IuJ$+wvxmdX_dS+7#c>eA=1_5w)ZoLCdNdN3Ov!7JjzNW6n~1g<Xhf7kV$=U(sHZ{-ho?rZrP`SeatnT)2o z{_jp86L_#_eP!`1H6kBK1c;UH09KcHqC+d?4HyGH zeSJqFCh61Dz8a#tp5Nx%O1RxBx)A{>K0SM^Cn^Pvd%6jH`YBbSR9Bxaae;TM`Y3@< z-!YKL*6!24o18uxv!vm z^*ZqBl@p0f{d{`AqjCYTCS1MQsgE#^W6abhAG<)m$lfF(qAY%wk)_A_%nrbkta5$1z!LM4!}4Glvk&@(?`!;4*|;yM=d)PH zGw*Edl!vDQOXy%CBFf@*1zBdVlUoKXBMZwc1eS}hIV{CNj88xO^T8(Qm#{KY&u7UW zcn8PW%gpG%fkm~Nh={UyMj^}DZsYm`OYDdNjslC{I}S^lDwAv1ctzlM*J~$`dOpjm zsm!|pord!QfF)r%5fNo^k4Bc)r%Ul0h^?Vi; zBW50RZ`2Owu_1{>M3lwh8nU$hkcj8YD)i61DX`cUb6EBnGjnpcrG7u4U&@A(dOpkd zj~DQM+1R@6ztAs3w26o)i|ut}@jYyQ3{24E_OHqX7VrOM`9BlHJ)?hA4U;)+#@pe3 zq+k`IB$7F9*ao4USmNi93z?&BAHE7TnnOFb#{Eb)y@oqO=2)#kRBFo{?*}uPWA=oG ziy$bR)*^D!GRN_E;mEhO@}?|Ed5o1JMIrx|V=_lpk7Mg0bLi<1sXxjbo;PRLK<04% zn@Eahj_eam=7<^S@(YHz{Q@E+3~`CfG0f#0ww3I-Z?{2Ytv*+Two-gG^KM}Nv4x)? zb97iiMCzG2B2x-`L*^K|o`^|ij_9g2m=sjP3p>N@*2A0#NXZWT{r}c^Fuhg|F2RSTcWbSmrcg`X%Y< zlzG6?_8O_@vs8@^z_r@A`NHqOGX5435oOtQ16e|z&gu&| z1^nGfbsezys*!p=i}e}i-N12wNBs9(wgwRqWig3I7UQgKD!|g!zuh{4C3-)HWl{p; z2|_L(!m~f$g^_wb%k!1KIL17c-{Kz40Z~Lml*QmCvh>tc#&-kT7tGo%u!J1surwIX zu*fa+#B%^I7?OHE%jVC_dyZar9{$2Hwwj2DvgqAHmU5?E_^*+0hy8~I7H=00%hqem zex#IQb#oYFJKab6-@gs|blvX%;yFBqrbMMS zpYCAC_;jmQ+wi*qEps9#&8HilWX5svy-O+J(--ZOB1Iu%eVC`-N_#caz^8YxBT|3l z({FZ}(Gz@nxFeAi_vugMv~U{TVZZV@3~{H!L`WFo5C^2SnWMW-nQ=)DZnteAL_ms9FFSo6 zl@yj3YJ*RYiXcjL_3089II`WTDd5v1t`gbWefozZ7tyEpl+||vpKcOQ1ncC}8|7`r z(bdmB%n;_jZHYuvbnc5~=05ic3l4%$*H0lb_4DZouM+0LT76J0B}14;kGXNy1pUv< zGc|>6`m15A`wSxW{52unhglPPyXwk7yNn)5L`2(V#vNP}9M{|68U5QVWD5lr$KxCp zHGgJ(Tev2@A+V_IA@zI~^P|k(Vb2V0Sz!6GpNNREOumaOX~7dRuwOoF)d(y1%dESsYcY2*n@6o_=>ouw8vy8dVWWnTHyV?Uwavl*8Wf_rxEZ2Q2(tyRr*RQw0 zlH<)`S>ez8zojs;_eEe)4<_||78@NVb97wt63=DyxOtvaxjY0Wkk zSmHxCEPoX5iteHwgL|IfGAWM<KHYHSYZZ8& z{@9AB)auh+8K0hb|HnZv*F|x41D_Y zBSh+te0qb7RfgcxgHI4iai4y-9plq;ss>MnA>QpQ5fX;Dgijxs=!*FwEbr<74YeYZxN-s`gDm4^xtBn2tIx0eIi@CPcOI>g+6`onYe23>0?ugV4Zxr z%$8j^FkVLV!E?eR(ut<%+*iQNeICd6_=8U$|Axra&!_uc7*`Ezf{$-OurQBCUgWF^ zBLkUtteceM@SpCg!K9wQCb-Kme(>mfUHmst!%IX&v|Tzs#5Eyn=kOQ6@@-wmIDsV~ zlEY#$lKEfYU*B}`%<;SCq@K_6@CNfO*|(~H@V~~nRzyUUrCll@2h2g;`jdJ-OW0?|cLbW1JHcA5I*f>jvb1`HEGv$soClVvE)KZ@ z%aIrki-R7sAF1YAQ#>4Ph6kzVv)t>=jIkSgC-eoDFfSq^%F;9iS$2*N!ZFr;N&I(# zWnVmpWuy|bjvoE-Sr__ckO`^hvoxB+?61guIN~6%d|O9EL|Gc7BFkB?WB$-DHA@<_ zi1?kKzTf7sJY3D}74h%%Y74MLnUH!ui|iieJDaARCv66ng{DMAl;tPGvR6$D&lFlT zVL~^7r7(fRa;k{AUjm|CV}a$zbW+b}IdqWOpB_*%c@MB0)+HjMEai`}U(^;xjD&vq z-qCi5z>@ud!xI0FVcA!zbqiP`yOVl8OZpNf*H#y3;=dET`VtXQmd|O(a=U{%?nk<> z95+Q^d78pu8PSE=8~;l0%>iK9tV-(nEQZ>V_|9gch5??Ho1#ucL|F=+Aj?U=tL?x9 zy$>w56D1;;tIcb^Wj2#MayEScG z!rpFX7RxsnuiR$7++v;Crism)$|}nAXw*dO5@wE~5t$i~Io#xlQe88L#04f?@$rJp z5vE9FYtJ0Y!8b5-6rY&$5HiQ}PDHRynZxRdJ!X!ob0HD1CN1kuG{xpV+rOUT+}G{)$;Xll)<?)oOj)TwqaFA@zKg zykuq%kH#hw+>@}P4-pY%$#{+|Q)})u1D0@yvq=Jr-y06g`Do_-Q&l@7Ctx{onAG!G zEC>F@!#D_r-+Cs%e@!K^2lPRAM{J=lj)5v{mxIepExYjjG6nz-XP%+u#{zz zdOl0n`~K){*UtSHv*2I3L`0M&{w1;;4^_u|Ut_<02Z5!mn8Q*tggJXWNmD%sSV~+- zJ)h;rW9EDKMHQ{BV2t(kAR?kH*Ipq@k?hb=Fb?Xn+ZPEe`@cx_>Ha2+Pv5tGrU^Lg z74k$$#Ha6B#O$?SGrIF?P>E3Ft59=RXvg^UgKn0u!KasXA}Y1{^r%S2r!RQZX9bw+ z-rb3uG@srW_T-oKuf2z^8|35lL~M zZaR*6H_&c~6~4w_HiifZLtMhATWByoeYko>DBLFxCv!z;E3N;@w9@4bY3bn8(`OQq zdiwN=R?QZJPrs*2#3X&XlY%3<>sfKhR&cv5(V<^$yoxCd{MxUpZ?+s4BC*opbZFhOs^&hSc-d zgvNuJ_xLaG6t#kO3AjT<#M(tBl4+OFbMClS=eBg+DzIdhb66%kWuCSCyR~o$upDSl z>iI0mri|~{I^^~&VENRQh={U;WnmvHZhpE4`sKi!yAA?NY9)u|#Vf`SCV!dj0xb56 zNj;yXXBP8rAnI}UabP*MjEIP`1iVHT9oLfu!16xp*GYlp&Myv&Z3|{UlD?1pCt!J( zN9y@3leRG)N~O!>eZVrNgouc;_`X4wrnfXY14~Yu5upN0w7s0fAt_ z$Exi~J)gxlo%vQd>0O5*z;gN!5fNoM@(x+THTI4MmJNj;^8}V-t{fJ(xeUuA!!t(E zFE_rCdOpjUE6lgb;aaY^-}GlS5fNo^dXFquQiFGaaY*;R-#_wqetPXDCvC6&sj1Ad z>{|>+Yk=3Mq6!KJ8T9Klzi=Iq&8#;4ya%oqqh-RLS&sm-UWMKL~o zQP}t-FxTVbiJUZ_9@mNS>AO;`JVDAJL5dWGJRiq6(D$B|tHGyNJt9(n zhdn2f;y&H0KjYKyDcOC7A-*?@2nj=6!l#=|V|=>xcm*{O*_gu>p{1_B*#Ffg@aayiKIz^QDX*)W1}KdqquIMQ5Nf5?Bj@L?SBAE+hUmo0?X%<9F|;<%Q)16 zo_58vt!>IlJ)dRICg!>A<4QdpVDbJ%L_}FE^N{6y))OaS*_EZeMqtS~!(ln{j5&)s zt>p*&CU#jKspqq7h+&?m2kvi;?>R0M6A@7s^AE`K?Jqri25t0WyOqF_;lp7$Y0dOY zY1B(ixL?-2BlUcippDF4`^*MLH=ti!3y6p)%Z7Yp3G)cmgnn67bICIw{_5zk-xyb+O`7B45GWn^kTuBkI#I+_OqAaUFBFl;NwLZWycfl8Lfh9hW!}8Og z$><6b9<)Rj15(dtDLuvPM_OmcoG+`liin7^EGs}32d5qpz!IUL5h<`lhH_X=STgT9 zDjpZ1PjA_Y)bm+NhB4n=$K5Ev^JTYnCnBOOOFtn?`Z!@yLKtHa8AL>sWkDgb=-wRo0!+{cJ!{#^zca_>2&tK4 z-cjb=!1@;3^&xWvEhkDMnPY1ZGiEyl9EO{TN^O}V zA)3h?7Quz@At=n>LFA-mj^>@2%%L51^gKu%w3Z@8Av=s?GRK6x;qH(*COZ(RKgt~5 zvFhcJIo`VvN%72~J(S5Ddt6qnfg!GQoCpa+Tq1MGOlC5N$@;6>Afj@bD?(do<1i+3 z$Y_+^fXrcdj)>GVbLbnNHiFEd97Mz zN5~we(L|}PnM2|Nn~XD#h0HN5j>y)YIjm--Vdl7Z-qIX0$JBd7uuhp{!%25Ef+G)9 z2g2Of{Sna=oBM1}F>{~0UG^Es9J`+pnfhf8^+z9kVSOu+wNw!1(bJbXYr;RN%--R$ z{Kx7r*4xUHdj6WwJB;yA5$#94gLWCzmWYV8i_GjIToWGW6s7>n%G~fC0?U!B9G0p5 znP<>SN1gJKrI^(7Suz$e=kVCejKMSdca#$mQI=`N$g=(X5nQVi+J4p)SoXzoSjs*z zK3y*0cp9)cD3N+TOZOs%Nm*79nqWTaKQ5Nmb$YQ15{|m6>PBAkOSPJiOSj@jM z`^3KgQg?%Xv6)Tk`7AoanEcdLHE09$OR^pj5oOW(f-K8AHctYU*xZX|0!wxxhvlLx zvwyE_dYh}jvcH(r^I5j(Fk>wBpcCfWCKW_Plx5IYWa)EWt~an8eDc*^V0rqG!(yb( z^h=i;XXAmzGn3TwS=1&mKHcy{7JfHyG>3?Yvh*uOmS262;&%g8TQxldmc&#J%iC-w zb97U1$NfmUdq_Q>#XO$z>He3}OJR&n-%mtDS$dTrOM@lv8i8?Gv&1%fqmD+sz z)HuecXAjH94f0cLiJUZ_9^I7j>EZ4P`@pBqJ0L}hLauOPe7fwaR#o8BWeyXmKl17K zuP*EdKE0s_kremoQ?(eMt~T9gH{7nx&k!MDh)ekN7ULP8z9c0S5Bk4#jw?c2DXk~t z(<@U~y$7Fe9ZW>(>C-jNtyu>?T_=KwN&573r4#6`hv@mL!R^-J8WE7<)6acOMy0vY zSC4{EUmZ`B>gv-aF7W7P!!q#c1_?yAcAq{dCj)(Y*LU-)z^5;IL)zAw*+rs&*vhnf2VTi!!^P6)8kvf;+7WpMPRv@!C@&n z&78wy+OP!k^@EqBp3idCmhl}E9~l|}%gtOOBFfUS99g_~{fGq?2ifdKSAIW-^2_9~ zbicx!{h7CH(jMp+`R1ga&mwEaoR1`{K5!hc{Afc&L|GIokR{o11nwP9z0|3rz~Y(B zVbNU0?2pyxY>nqK9*H6Kd=~eSjPJN>+UpCjY`;T9L|IyVN0yv1ddko*liU{e7g(Hg zI4lo;G0)SJ{`rUtjKwKZ&u7^o6NP(tW+~{J0E>|?5fNo+@&j2KU4I$|EXo%Sj}=&K z^EoV2A2Occg0gNO7-LPMNj;zC_FE=DDOKoWMlXvaBBCrZmB?aIs=OFjlEV_`2`puw zI4ld(n7vio<)-2p*T1fidOnLuIrDAG+@3!ULBHhRAR?kHl?=iH~d_cAQG2diHK%P|ciBFa)$h5e%9({~!MMA?k76j(A# zIV@)`F#TfCFVY5B${a~OpT+%<58f~3XHz_YMaGSYh_V#_M3xU$H`Blb&6TYT7g*l@ zFU$X#AnqCcQx-CxbT0LMWesUzpgd6$$sDDJn14Y2nJO=I2-%r#nc z4l;*zC!$hY=D6p@WRAYgI~GDv7^_0$q-Bn^YRvrd;)&^WkWx~UB1Iveq%xUfQ0aaL z$Q;hn<$s9ILDtHvfV{IZN3~`Cfab^;eIZX8`WZ^zB zp2QWQtu%8ClR2(D7~B;yhwTg^QqRocc&1fj$Q+&L5i!ZkF|+$=ObW9ujhqg*+s(y9 zKuYE?x$zX0ri}{7hRpHRh$z)Hb4Xm^2!pHTkU7j&5!u=^M+3!d%p4(yay=n)EZ;x` z>y$YZz0P3fcyVMA{ufNyf@q4(eYT&Oxo_IsMV^p3jCT>4`ehCyyXQKvCMYcLbyt{2 zQ!6-YLUufJ-peoZu6VX}tM#Owza}hr!Tf(}ne#Z@lMrA|M8w)fCchfj1l34W{7>wc z)=M%3mOE7(mY^|A<}lW-421i3bSF~JXR)_nzJX9pe>4Hsx0&6Eh$zduU&ylhlNRpj zc8PTVB(Ox+a9DPKx`IQkU`93i4(C`>&u3|WmO1|~Zn3i#u$0{;BBCsrHOSI(>^1$pC$7Olb^p$MKSg=o&~?6<;>m!%dtZo7MmW7C-~YkY9O#&`-{}`S@!BP-wkBU z-Y^ANhIAz&qAaUqv0tiPUO2!Qs|t4-A+R_e=CFJVVeXe^g~gA7#s3qhh_V&1 | tee -a /logs/wmt_pytorch_09-13-2026-09-42-13.log +W0913 09:42:43.155000 9 site-packages/torch/distributed/run.py:803] +W0913 09:42:43.155000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0913 09:42:43.155000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0913 09:42:43.155000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-13 09:42:59.840360: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-13 09:42:59.840363: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-13 09:42:59.840360: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-13 09:42:59.840360: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789292580.482678 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789292580.482692 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789292580.482680 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789292580.482683 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789292580.593819 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789292580.593813 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789292580.593823 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789292580.593827 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789292582.338231 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789292582.338247 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789292582.338248 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789292582.338275 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789292582.338280 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789292582.338257 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789292582.338280 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789292582.338282 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789292582.338283 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789292582.338284 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789292582.338285 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789292582.338285 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789292582.338285 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789292582.338286 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789292582.338288 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789292582.338290 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789292614.886782 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789292614.886782 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789292614.886792 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789292614.897925 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W913 09:43:47.507126539 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0913 09:43:50.278599 139802006037696 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/wmt_pytorch. +I0913 09:43:50.278593 140342140011712 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/wmt_pytorch. +I0913 09:43:50.278592 140228673635520 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/wmt_pytorch. +I0913 09:43:50.278630 140125701207232 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/wmt_pytorch. +I0913 09:43:50.542884 139802006037696 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_0/wmt_pytorch/trial_1. +I0913 09:43:50.818061 139802006037696 submission_runner.py:242] Initializing dataset. +I0913 09:43:50.818240 139802006037696 submission_runner.py:251] Initializing model. +I0913 09:44:03.964022 139802006037696 submission_runner.py:290] Performing `torch.compile`. +I0913 09:44:08.107244 139802006037696 submission_runner.py:294] Initializing optimizer. +I0913 09:44:08.108158 139802006037696 submission_runner.py:299] Initializing metrics bundle. +I0913 09:44:08.108310 139802006037696 submission_runner.py:321] Initializing checkpoint and logger. +I0913 09:44:08.110139 139802006037696 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_0/wmt_pytorch/trial_1/meta_data_0.json. +I0913 09:44:08.110208 140342140011712 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0913 09:44:08.110209 140125701207232 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0913 09:44:08.110229 140228673635520 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0913 09:44:08.110346 139802006037696 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0913 09:44:08.110353 140125701207232 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0913 09:44:08.110357 140342140011712 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0913 09:44:08.110373 140228673635520 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0913 09:44:08.110409 139802006037696 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0913 09:44:08.568071 139802006037696 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_0/wmt_pytorch/trial_1/flags_0.json. +I0913 09:44:08.683043 139802006037696 submission_runner.py:359] Starting training loop. +I0913 09:44:10.162969 139802006037696 dataset_info.py:707] Load dataset info from /data/wmt/wmt17_translate/de-en/1.0.0 +I0913 09:44:10.175633 139802006037696 dataset_info.py:793] For 'wmt17_translate/de-en/1.0.0': fields info.[splits, supervised_keys] differ on disk and in the code. Keeping the one from code. +I0913 09:44:10.202576 139802006037696 reader.py:262] Creating a tf.data.Dataset reading 16 files located in folders: /data/wmt/wmt17_translate/de-en/1.0.0. +I0913 09:44:10.316531 139802006037696 logging_logger.py:49] Constructing tf.data.Dataset wmt17_translate for split train, from /data/wmt/wmt17_translate/de-en/1.0.0 +[rank1]:W0913 09:44:12.830000 39 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank3]:W0913 09:44:12.830000 41 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank2]:W0913 09:44:12.830000 40 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank0]:W0913 09:44:13.602000 38 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +I0913 09:45:24.718703 139777193981696 logging_writer.py:48] [0] global_step=0, grad_norm=0.5, loss=11.7563 +I0913 09:45:25.002798 139802006037696 submission.py:307] 0) loss = 11.756, grad_norm = 0.500 +I0913 09:45:25.694120 139802006037696 spec.py:333] Evaluating on the training split. +I0913 09:45:25.696110 139802006037696 dataset_info.py:707] Load dataset info from /data/wmt/wmt17_translate/de-en/1.0.0 +I0913 09:45:25.698222 139802006037696 dataset_info.py:793] For 'wmt17_translate/de-en/1.0.0': fields info.[splits, supervised_keys] differ on disk and in the code. Keeping the one from code. +I0913 09:45:25.699187 139802006037696 reader.py:262] Creating a tf.data.Dataset reading 16 files located in folders: /data/wmt/wmt17_translate/de-en/1.0.0. +I0913 09:45:25.733480 139802006037696 logging_logger.py:49] Constructing tf.data.Dataset wmt17_translate for split train, from /data/wmt/wmt17_translate/de-en/1.0.0 +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +I0913 09:46:01.316235 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 09:48:17.523988 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 09:48:17.588430 139802006037696 dataset_info.py:707] Load dataset info from /data/wmt/wmt14_translate/de-en/1.0.0 +I0913 09:48:17.658563 139802006037696 dataset_info.py:793] For 'wmt14_translate/de-en/1.0.0': fields info.[splits, supervised_keys] differ on disk and in the code. Keeping the one from code. +I0913 09:48:17.659365 139802006037696 reader.py:262] Creating a tf.data.Dataset reading 1 files located in folders: /data/wmt/wmt14_translate/de-en/1.0.0. +I0913 09:48:17.695574 139802006037696 logging_logger.py:49] Constructing tf.data.Dataset wmt14_translate for split validation, from /data/wmt/wmt14_translate/de-en/1.0.0 +I0913 09:48:20.055020 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 09:50:32.095145 139802006037696 spec.py:363] Evaluating on the test split. +I0913 09:50:32.097463 139802006037696 dataset_info.py:707] Load dataset info from /data/wmt/wmt14_translate/de-en/1.0.0 +I0913 09:50:32.099794 139802006037696 dataset_info.py:793] For 'wmt14_translate/de-en/1.0.0': fields info.[splits, supervised_keys] differ on disk and in the code. Keeping the one from code. +I0913 09:50:32.100610 139802006037696 reader.py:262] Creating a tf.data.Dataset reading 1 files located in folders: /data/wmt/wmt14_translate/de-en/1.0.0. +I0913 09:50:32.136340 139802006037696 logging_logger.py:49] Constructing tf.data.Dataset wmt14_translate for split test, from /data/wmt/wmt14_translate/de-en/1.0.0 +I0913 09:50:34.448883 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 09:52:46.646568 139802006037696 submission_runner.py:516] Time since start: 517.96s, Step: 1, {'train/accuracy': 0.0006042089422923459, 'train/loss': 12.051779566337581, 'train/bleu': 0.0, 'validation/accuracy': 0.00048356498989473163, 'validation/loss': 12.05185847044674, 'validation/bleu': 0.0, 'validation/num_examples': 3000, 'test/accuracy': 0.0007088489919237697, 'test/loss': 12.044632792981233, 'test/bleu': 0.0, 'test/num_examples': 3003, 'score': 76.32085061073303, 'total_duration': 517.9636764526367, 'accumulated_submission_time': 76.32085061073303, 'accumulated_eval_time': 440.9525320529938, 'accumulated_logging_time': 0} +I0913 09:52:46.671370 139773553383168 logging_writer.py:48] [1] accumulated_eval_time=440.953, accumulated_logging_time=0, accumulated_submission_time=76.3209, global_step=1, preemption_count=0, score=76.3209, test/accuracy=0.000708849, test/bleu=0, test/loss=12.0446, test/num_examples=3003, total_duration=517.964, train/accuracy=0.000604209, train/bleu=0, train/loss=12.0518, validation/accuracy=0.000483565, validation/bleu=0, validation/loss=12.0519, validation/num_examples=3000 +I0913 09:52:47.613366 139773544990464 logging_writer.py:48] [1] global_step=1, grad_norm=0.5, loss=11.7447 +I0913 09:52:47.616377 139802006037696 submission.py:307] 1) loss = 11.745, grad_norm = 0.500 +I0913 09:52:47.784703 139773553383168 logging_writer.py:48] [2] global_step=2, grad_norm=0.5, loss=11.74 +I0913 09:52:47.787577 139802006037696 submission.py:307] 2) loss = 11.740, grad_norm = 0.500 +I0913 09:52:47.955988 139773544990464 logging_writer.py:48] [3] global_step=3, grad_norm=0.5, loss=11.7389 +I0913 09:52:47.958954 139802006037696 submission.py:307] 3) loss = 11.739, grad_norm = 0.500 +I0913 09:52:48.126898 139773553383168 logging_writer.py:48] [4] global_step=4, grad_norm=0.5, loss=11.703 +I0913 09:52:48.129729 139802006037696 submission.py:307] 4) loss = 11.703, grad_norm = 0.500 +I0913 09:52:48.298053 139773544990464 logging_writer.py:48] [5] global_step=5, grad_norm=0.5, loss=11.6842 +I0913 09:52:48.300910 139802006037696 submission.py:307] 5) loss = 11.684, grad_norm = 0.500 +I0913 09:52:48.469523 139773553383168 logging_writer.py:48] [6] global_step=6, grad_norm=0.5, loss=11.6628 +I0913 09:52:48.472414 139802006037696 submission.py:307] 6) loss = 11.663, grad_norm = 0.500 +I0913 09:52:48.640353 139773544990464 logging_writer.py:48] [7] global_step=7, grad_norm=0.5, loss=11.6169 +I0913 09:52:48.643230 139802006037696 submission.py:307] 7) loss = 11.617, grad_norm = 0.500 +I0913 09:52:48.811556 139773553383168 logging_writer.py:48] [8] global_step=8, grad_norm=0.5, loss=11.5832 +I0913 09:52:48.814512 139802006037696 submission.py:307] 8) loss = 11.583, grad_norm = 0.500 +I0913 09:52:48.983727 139773544990464 logging_writer.py:48] [9] global_step=9, grad_norm=0.5, loss=11.5352 +I0913 09:52:48.986821 139802006037696 submission.py:307] 9) loss = 11.535, grad_norm = 0.500 +I0913 09:52:49.154510 139773553383168 logging_writer.py:48] [10] global_step=10, grad_norm=0.5, loss=11.4714 +I0913 09:52:49.157330 139802006037696 submission.py:307] 10) loss = 11.471, grad_norm = 0.500 +I0913 09:52:49.325563 139773544990464 logging_writer.py:48] [11] global_step=11, grad_norm=0.5, loss=11.4339 +I0913 09:52:49.328443 139802006037696 submission.py:307] 11) loss = 11.434, grad_norm = 0.500 +I0913 09:52:49.496160 139773553383168 logging_writer.py:48] [12] global_step=12, grad_norm=0.5, loss=11.3729 +I0913 09:52:49.499096 139802006037696 submission.py:307] 12) loss = 11.373, grad_norm = 0.500 +I0913 09:52:49.667586 139773544990464 logging_writer.py:48] [13] global_step=13, grad_norm=0.5, loss=11.3232 +I0913 09:52:49.670538 139802006037696 submission.py:307] 13) loss = 11.323, grad_norm = 0.500 +I0913 09:52:49.837884 139773553383168 logging_writer.py:48] [14] global_step=14, grad_norm=0.5, loss=11.2433 +I0913 09:52:49.840650 139802006037696 submission.py:307] 14) loss = 11.243, grad_norm = 0.500 +I0913 09:52:50.008143 139773544990464 logging_writer.py:48] [15] global_step=15, grad_norm=0.5, loss=11.1749 +I0913 09:52:50.010993 139802006037696 submission.py:307] 15) loss = 11.175, grad_norm = 0.500 +I0913 09:52:50.178220 139773553383168 logging_writer.py:48] [16] global_step=16, grad_norm=0.5, loss=11.109 +I0913 09:52:50.181058 139802006037696 submission.py:307] 16) loss = 11.109, grad_norm = 0.500 +I0913 09:52:50.349038 139773544990464 logging_writer.py:48] [17] global_step=17, grad_norm=0.5, loss=11.0378 +I0913 09:52:50.351967 139802006037696 submission.py:307] 17) loss = 11.038, grad_norm = 0.500 +I0913 09:52:50.519723 139773553383168 logging_writer.py:48] [18] global_step=18, grad_norm=0.5, loss=10.9675 +I0913 09:52:50.522631 139802006037696 submission.py:307] 18) loss = 10.967, grad_norm = 0.500 +I0913 09:52:50.690739 139773544990464 logging_writer.py:48] [19] global_step=19, grad_norm=0.5, loss=10.9017 +I0913 09:52:50.693661 139802006037696 submission.py:307] 19) loss = 10.902, grad_norm = 0.500 +I0913 09:52:50.862057 139773553383168 logging_writer.py:48] [20] global_step=20, grad_norm=0.5, loss=10.828 +I0913 09:52:50.864976 139802006037696 submission.py:307] 20) loss = 10.828, grad_norm = 0.500 +I0913 09:52:51.033637 139773544990464 logging_writer.py:48] [21] global_step=21, grad_norm=0.5, loss=10.7574 +I0913 09:52:51.036517 139802006037696 submission.py:307] 21) loss = 10.757, grad_norm = 0.500 +I0913 09:52:51.204571 139773553383168 logging_writer.py:48] [22] global_step=22, grad_norm=0.5, loss=10.6825 +I0913 09:52:51.207383 139802006037696 submission.py:307] 22) loss = 10.682, grad_norm = 0.500 +I0913 09:52:51.376113 139773544990464 logging_writer.py:48] [23] global_step=23, grad_norm=0.5, loss=10.6104 +I0913 09:52:51.378980 139802006037696 submission.py:307] 23) loss = 10.610, grad_norm = 0.500 +I0913 09:52:51.547859 139773553383168 logging_writer.py:48] [24] global_step=24, grad_norm=0.5, loss=10.5568 +I0913 09:52:51.550744 139802006037696 submission.py:307] 24) loss = 10.557, grad_norm = 0.500 +I0913 09:52:51.719407 139773544990464 logging_writer.py:48] [25] global_step=25, grad_norm=0.5, loss=10.4865 +I0913 09:52:51.722467 139802006037696 submission.py:307] 25) loss = 10.487, grad_norm = 0.500 +I0913 09:52:51.890880 139773553383168 logging_writer.py:48] [26] global_step=26, grad_norm=0.5, loss=10.4073 +I0913 09:52:51.893794 139802006037696 submission.py:307] 26) loss = 10.407, grad_norm = 0.500 +I0913 09:52:52.062127 139773544990464 logging_writer.py:48] [27] global_step=27, grad_norm=0.5, loss=10.328 +I0913 09:52:52.065001 139802006037696 submission.py:307] 27) loss = 10.328, grad_norm = 0.500 +I0913 09:52:52.233206 139773553383168 logging_writer.py:48] [28] global_step=28, grad_norm=0.5, loss=10.2566 +I0913 09:52:52.236135 139802006037696 submission.py:307] 28) loss = 10.257, grad_norm = 0.500 +I0913 09:52:52.404020 139773544990464 logging_writer.py:48] [29] global_step=29, grad_norm=0.5, loss=10.2046 +I0913 09:52:52.406960 139802006037696 submission.py:307] 29) loss = 10.205, grad_norm = 0.500 +I0913 09:52:52.575910 139773553383168 logging_writer.py:48] [30] global_step=30, grad_norm=0.5, loss=10.1139 +I0913 09:52:52.578907 139802006037696 submission.py:307] 30) loss = 10.114, grad_norm = 0.500 +I0913 09:52:52.747149 139773544990464 logging_writer.py:48] [31] global_step=31, grad_norm=0.5, loss=10.0653 +I0913 09:52:52.750032 139802006037696 submission.py:307] 31) loss = 10.065, grad_norm = 0.500 +I0913 09:52:52.918616 139773553383168 logging_writer.py:48] [32] global_step=32, grad_norm=0.5, loss=10.0031 +I0913 09:52:52.921411 139802006037696 submission.py:307] 32) loss = 10.003, grad_norm = 0.500 +I0913 09:52:53.089070 139773544990464 logging_writer.py:48] [33] global_step=33, grad_norm=0.5, loss=9.93625 +I0913 09:52:53.092014 139802006037696 submission.py:307] 33) loss = 9.936, grad_norm = 0.500 +I0913 09:52:53.260113 139773553383168 logging_writer.py:48] [34] global_step=34, grad_norm=0.5, loss=9.86102 +I0913 09:52:53.263102 139802006037696 submission.py:307] 34) loss = 9.861, grad_norm = 0.500 +I0913 09:52:53.430817 139773544990464 logging_writer.py:48] [35] global_step=35, grad_norm=0.5, loss=9.81305 +I0913 09:52:53.433792 139802006037696 submission.py:307] 35) loss = 9.813, grad_norm = 0.500 +I0913 09:52:53.602334 139773553383168 logging_writer.py:48] [36] global_step=36, grad_norm=0.5, loss=9.76802 +I0913 09:52:53.605199 139802006037696 submission.py:307] 36) loss = 9.768, grad_norm = 0.500 +I0913 09:52:53.773449 139773544990464 logging_writer.py:48] [37] global_step=37, grad_norm=0.5, loss=9.7033 +I0913 09:52:53.776266 139802006037696 submission.py:307] 37) loss = 9.703, grad_norm = 0.500 +I0913 09:52:53.944019 139773553383168 logging_writer.py:48] [38] global_step=38, grad_norm=0.5, loss=9.63778 +I0913 09:52:53.946957 139802006037696 submission.py:307] 38) loss = 9.638, grad_norm = 0.500 +I0913 09:52:54.115075 139773544990464 logging_writer.py:48] [39] global_step=39, grad_norm=0.5, loss=9.60707 +I0913 09:52:54.118085 139802006037696 submission.py:307] 39) loss = 9.607, grad_norm = 0.500 +I0913 09:52:54.286290 139773553383168 logging_writer.py:48] [40] global_step=40, grad_norm=0.5, loss=9.50353 +I0913 09:52:54.289111 139802006037696 submission.py:307] 40) loss = 9.504, grad_norm = 0.500 +I0913 09:52:54.457561 139773544990464 logging_writer.py:48] [41] global_step=41, grad_norm=0.5, loss=9.48126 +I0913 09:52:54.460601 139802006037696 submission.py:307] 41) loss = 9.481, grad_norm = 0.500 +I0913 09:52:54.629636 139773553383168 logging_writer.py:48] [42] global_step=42, grad_norm=0.5, loss=9.41728 +I0913 09:52:54.632551 139802006037696 submission.py:307] 42) loss = 9.417, grad_norm = 0.500 +I0913 09:52:54.801399 139773544990464 logging_writer.py:48] [43] global_step=43, grad_norm=0.5, loss=9.34829 +I0913 09:52:54.804340 139802006037696 submission.py:307] 43) loss = 9.348, grad_norm = 0.500 +I0913 09:52:54.972645 139773553383168 logging_writer.py:48] [44] global_step=44, grad_norm=0.5, loss=9.30149 +I0913 09:52:54.975453 139802006037696 submission.py:307] 44) loss = 9.301, grad_norm = 0.500 +I0913 09:52:55.144569 139773544990464 logging_writer.py:48] [45] global_step=45, grad_norm=0.5, loss=9.26985 +I0913 09:52:55.147451 139802006037696 submission.py:307] 45) loss = 9.270, grad_norm = 0.500 +I0913 09:52:55.316661 139773553383168 logging_writer.py:48] [46] global_step=46, grad_norm=0.5, loss=9.21878 +I0913 09:52:55.319511 139802006037696 submission.py:307] 46) loss = 9.219, grad_norm = 0.500 +I0913 09:52:55.488037 139773544990464 logging_writer.py:48] [47] global_step=47, grad_norm=0.499999, loss=9.18246 +I0913 09:52:55.490946 139802006037696 submission.py:307] 47) loss = 9.182, grad_norm = 0.500 +I0913 09:52:55.659612 139773553383168 logging_writer.py:48] [48] global_step=48, grad_norm=0.499999, loss=9.1409 +I0913 09:52:55.662516 139802006037696 submission.py:307] 48) loss = 9.141, grad_norm = 0.500 +I0913 09:52:55.831652 139773544990464 logging_writer.py:48] [49] global_step=49, grad_norm=0.499999, loss=9.12004 +I0913 09:52:55.834578 139802006037696 submission.py:307] 49) loss = 9.120, grad_norm = 0.500 +I0913 09:52:56.003263 139773553383168 logging_writer.py:48] [50] global_step=50, grad_norm=0.499999, loss=9.07515 +I0913 09:52:56.006179 139802006037696 submission.py:307] 50) loss = 9.075, grad_norm = 0.500 +I0913 09:52:56.174761 139773544990464 logging_writer.py:48] [51] global_step=51, grad_norm=0.499999, loss=9.02138 +I0913 09:52:56.177646 139802006037696 submission.py:307] 51) loss = 9.021, grad_norm = 0.500 +I0913 09:52:56.347066 139773553383168 logging_writer.py:48] [52] global_step=52, grad_norm=0.499999, loss=8.9656 +I0913 09:52:56.349982 139802006037696 submission.py:307] 52) loss = 8.966, grad_norm = 0.500 +I0913 09:52:56.518933 139773544990464 logging_writer.py:48] [53] global_step=53, grad_norm=0.499999, loss=8.97992 +I0913 09:52:56.521843 139802006037696 submission.py:307] 53) loss = 8.980, grad_norm = 0.500 +I0913 09:52:56.690422 139773553383168 logging_writer.py:48] [54] global_step=54, grad_norm=0.499999, loss=8.95882 +I0913 09:52:56.693276 139802006037696 submission.py:307] 54) loss = 8.959, grad_norm = 0.500 +I0913 09:52:56.861788 139773544990464 logging_writer.py:48] [55] global_step=55, grad_norm=0.499999, loss=8.92498 +I0913 09:52:56.864778 139802006037696 submission.py:307] 55) loss = 8.925, grad_norm = 0.500 +I0913 09:52:57.034007 139773553383168 logging_writer.py:48] [56] global_step=56, grad_norm=0.499999, loss=8.87156 +I0913 09:52:57.036973 139802006037696 submission.py:307] 56) loss = 8.872, grad_norm = 0.500 +I0913 09:52:57.205992 139773544990464 logging_writer.py:48] [57] global_step=57, grad_norm=0.499999, loss=8.83127 +I0913 09:52:57.209109 139802006037696 submission.py:307] 57) loss = 8.831, grad_norm = 0.500 +I0913 09:52:57.378359 139773553383168 logging_writer.py:48] [58] global_step=58, grad_norm=0.499999, loss=8.82303 +I0913 09:52:57.381241 139802006037696 submission.py:307] 58) loss = 8.823, grad_norm = 0.500 +I0913 09:52:57.549830 139773544990464 logging_writer.py:48] [59] global_step=59, grad_norm=0.489358, loss=8.83859 +I0913 09:52:57.552761 139802006037696 submission.py:307] 59) loss = 8.839, grad_norm = 0.489 +I0913 09:52:57.721586 139773553383168 logging_writer.py:48] [60] global_step=60, grad_norm=0.46306, loss=8.74436 +I0913 09:52:57.724519 139802006037696 submission.py:307] 60) loss = 8.744, grad_norm = 0.463 +I0913 09:52:57.892800 139773544990464 logging_writer.py:48] [61] global_step=61, grad_norm=0.43619, loss=8.77637 +I0913 09:52:57.895777 139802006037696 submission.py:307] 61) loss = 8.776, grad_norm = 0.436 +I0913 09:52:58.064275 139773553383168 logging_writer.py:48] [62] global_step=62, grad_norm=0.415374, loss=8.73356 +I0913 09:52:58.067214 139802006037696 submission.py:307] 62) loss = 8.734, grad_norm = 0.415 +I0913 09:52:58.235737 139773544990464 logging_writer.py:48] [63] global_step=63, grad_norm=0.382657, loss=8.73622 +I0913 09:52:58.238704 139802006037696 submission.py:307] 63) loss = 8.736, grad_norm = 0.383 +I0913 09:52:58.407527 139773553383168 logging_writer.py:48] [64] global_step=64, grad_norm=0.371603, loss=8.68712 +I0913 09:52:58.410399 139802006037696 submission.py:307] 64) loss = 8.687, grad_norm = 0.372 +I0913 09:52:58.578878 139773544990464 logging_writer.py:48] [65] global_step=65, grad_norm=0.352186, loss=8.72379 +I0913 09:52:58.581919 139802006037696 submission.py:307] 65) loss = 8.724, grad_norm = 0.352 +I0913 09:52:58.750969 139773553383168 logging_writer.py:48] [66] global_step=66, grad_norm=0.341268, loss=8.6216 +I0913 09:52:58.753869 139802006037696 submission.py:307] 66) loss = 8.622, grad_norm = 0.341 +I0913 09:52:58.921690 139773544990464 logging_writer.py:48] [67] global_step=67, grad_norm=0.328861, loss=8.64882 +I0913 09:52:58.924527 139802006037696 submission.py:307] 67) loss = 8.649, grad_norm = 0.329 +I0913 09:52:59.092509 139773553383168 logging_writer.py:48] [68] global_step=68, grad_norm=0.309134, loss=8.62639 +I0913 09:52:59.095381 139802006037696 submission.py:307] 68) loss = 8.626, grad_norm = 0.309 +I0913 09:52:59.263886 139773544990464 logging_writer.py:48] [69] global_step=69, grad_norm=0.303449, loss=8.60379 +I0913 09:52:59.266917 139802006037696 submission.py:307] 69) loss = 8.604, grad_norm = 0.303 +I0913 09:52:59.436156 139773553383168 logging_writer.py:48] [70] global_step=70, grad_norm=0.303755, loss=8.58438 +I0913 09:52:59.439017 139802006037696 submission.py:307] 70) loss = 8.584, grad_norm = 0.304 +I0913 09:52:59.607730 139773544990464 logging_writer.py:48] [71] global_step=71, grad_norm=0.278121, loss=8.6016 +I0913 09:52:59.610645 139802006037696 submission.py:307] 71) loss = 8.602, grad_norm = 0.278 +I0913 09:52:59.779454 139773553383168 logging_writer.py:48] [72] global_step=72, grad_norm=0.264038, loss=8.55858 +I0913 09:52:59.782364 139802006037696 submission.py:307] 72) loss = 8.559, grad_norm = 0.264 +I0913 09:52:59.951283 139773544990464 logging_writer.py:48] [73] global_step=73, grad_norm=0.262936, loss=8.59861 +I0913 09:52:59.954144 139802006037696 submission.py:307] 73) loss = 8.599, grad_norm = 0.263 +I0913 09:53:00.123485 139773553383168 logging_writer.py:48] [74] global_step=74, grad_norm=0.26319, loss=8.55649 +I0913 09:53:00.126346 139802006037696 submission.py:307] 74) loss = 8.556, grad_norm = 0.263 +I0913 09:53:00.294551 139773544990464 logging_writer.py:48] [75] global_step=75, grad_norm=0.246045, loss=8.55223 +I0913 09:53:00.297475 139802006037696 submission.py:307] 75) loss = 8.552, grad_norm = 0.246 +I0913 09:53:00.466649 139773553383168 logging_writer.py:48] [76] global_step=76, grad_norm=0.238813, loss=8.50806 +I0913 09:53:00.469523 139802006037696 submission.py:307] 76) loss = 8.508, grad_norm = 0.239 +I0913 09:53:00.637594 139773544990464 logging_writer.py:48] [77] global_step=77, grad_norm=0.247021, loss=8.51851 +I0913 09:53:00.640510 139802006037696 submission.py:307] 77) loss = 8.519, grad_norm = 0.247 +I0913 09:53:00.809500 139773553383168 logging_writer.py:48] [78] global_step=78, grad_norm=0.232612, loss=8.54103 +I0913 09:53:00.812472 139802006037696 submission.py:307] 78) loss = 8.541, grad_norm = 0.233 +I0913 09:53:00.980334 139773544990464 logging_writer.py:48] [79] global_step=79, grad_norm=0.231388, loss=8.50938 +I0913 09:53:00.983153 139802006037696 submission.py:307] 79) loss = 8.509, grad_norm = 0.231 +I0913 09:53:01.152273 139773553383168 logging_writer.py:48] [80] global_step=80, grad_norm=0.228978, loss=8.53021 +I0913 09:53:01.155215 139802006037696 submission.py:307] 80) loss = 8.530, grad_norm = 0.229 +I0913 09:53:01.324196 139773544990464 logging_writer.py:48] [81] global_step=81, grad_norm=0.229463, loss=8.48234 +I0913 09:53:01.327105 139802006037696 submission.py:307] 81) loss = 8.482, grad_norm = 0.229 +I0913 09:53:01.495551 139773553383168 logging_writer.py:48] [82] global_step=82, grad_norm=0.223127, loss=8.4917 +I0913 09:53:01.498430 139802006037696 submission.py:307] 82) loss = 8.492, grad_norm = 0.223 +I0913 09:53:01.666823 139773544990464 logging_writer.py:48] [83] global_step=83, grad_norm=0.218633, loss=8.50015 +I0913 09:53:01.669668 139802006037696 submission.py:307] 83) loss = 8.500, grad_norm = 0.219 +I0913 09:53:01.838539 139773553383168 logging_writer.py:48] [84] global_step=84, grad_norm=0.224571, loss=8.4558 +I0913 09:53:01.841463 139802006037696 submission.py:307] 84) loss = 8.456, grad_norm = 0.225 +I0913 09:53:02.009858 139773544990464 logging_writer.py:48] [85] global_step=85, grad_norm=0.204582, loss=8.50424 +I0913 09:53:02.012801 139802006037696 submission.py:307] 85) loss = 8.504, grad_norm = 0.205 +I0913 09:53:02.182283 139773553383168 logging_writer.py:48] [86] global_step=86, grad_norm=0.208346, loss=8.4654 +I0913 09:53:02.185210 139802006037696 submission.py:307] 86) loss = 8.465, grad_norm = 0.208 +I0913 09:53:02.354293 139773544990464 logging_writer.py:48] [87] global_step=87, grad_norm=0.211052, loss=8.43173 +I0913 09:53:02.357225 139802006037696 submission.py:307] 87) loss = 8.432, grad_norm = 0.211 +I0913 09:53:02.526241 139773553383168 logging_writer.py:48] [88] global_step=88, grad_norm=0.206687, loss=8.44124 +I0913 09:53:02.529192 139802006037696 submission.py:307] 88) loss = 8.441, grad_norm = 0.207 +I0913 09:53:02.697746 139773544990464 logging_writer.py:48] [89] global_step=89, grad_norm=0.209209, loss=8.41497 +I0913 09:53:02.702628 139802006037696 submission.py:307] 89) loss = 8.415, grad_norm = 0.209 +I0913 09:53:02.872042 139773553383168 logging_writer.py:48] [90] global_step=90, grad_norm=0.21398, loss=8.42699 +I0913 09:53:02.875102 139802006037696 submission.py:307] 90) loss = 8.427, grad_norm = 0.214 +I0913 09:53:03.043620 139773544990464 logging_writer.py:48] [91] global_step=91, grad_norm=0.211809, loss=8.46079 +I0913 09:53:03.046750 139802006037696 submission.py:307] 91) loss = 8.461, grad_norm = 0.212 +I0913 09:53:03.215734 139773553383168 logging_writer.py:48] [92] global_step=92, grad_norm=0.207807, loss=8.42408 +I0913 09:53:03.218602 139802006037696 submission.py:307] 92) loss = 8.424, grad_norm = 0.208 +I0913 09:53:03.386611 139773544990464 logging_writer.py:48] [93] global_step=93, grad_norm=0.206256, loss=8.42784 +I0913 09:53:03.389486 139802006037696 submission.py:307] 93) loss = 8.428, grad_norm = 0.206 +I0913 09:53:03.558514 139773553383168 logging_writer.py:48] [94] global_step=94, grad_norm=0.215245, loss=8.4231 +I0913 09:53:03.561389 139802006037696 submission.py:307] 94) loss = 8.423, grad_norm = 0.215 +I0913 09:53:03.730668 139773544990464 logging_writer.py:48] [95] global_step=95, grad_norm=0.204298, loss=8.39332 +I0913 09:53:03.733500 139802006037696 submission.py:307] 95) loss = 8.393, grad_norm = 0.204 +I0913 09:53:03.902198 139773553383168 logging_writer.py:48] [96] global_step=96, grad_norm=0.206524, loss=8.35753 +I0913 09:53:03.905070 139802006037696 submission.py:307] 96) loss = 8.358, grad_norm = 0.207 +I0913 09:53:04.073626 139773544990464 logging_writer.py:48] [97] global_step=97, grad_norm=0.195891, loss=8.38388 +I0913 09:53:04.076411 139802006037696 submission.py:307] 97) loss = 8.384, grad_norm = 0.196 +I0913 09:53:04.245584 139773553383168 logging_writer.py:48] [98] global_step=98, grad_norm=0.205584, loss=8.41147 +I0913 09:53:04.248494 139802006037696 submission.py:307] 98) loss = 8.411, grad_norm = 0.206 +I0913 09:53:04.416724 139773544990464 logging_writer.py:48] [99] global_step=99, grad_norm=0.190392, loss=8.42881 +I0913 09:53:04.419700 139802006037696 submission.py:307] 99) loss = 8.429, grad_norm = 0.190 +I0913 09:53:04.588737 139773553383168 logging_writer.py:48] [100] global_step=100, grad_norm=0.193781, loss=8.37338 +I0913 09:53:04.591631 139802006037696 submission.py:307] 100) loss = 8.373, grad_norm = 0.194 +I0913 09:54:06.653643 139773544990464 logging_writer.py:48] [500] global_step=500, grad_norm=0.499999, loss=6.71127 +I0913 09:54:06.656838 139802006037696 submission.py:307] 500) loss = 6.711, grad_norm = 0.500 +I0913 09:55:23.788388 139773553383168 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.499999, loss=5.55056 +I0913 09:55:23.791787 139802006037696 submission.py:307] 1000) loss = 5.551, grad_norm = 0.500 +I0913 09:56:40.935437 139773544990464 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.499999, loss=4.5845 +I0913 09:56:40.938647 139802006037696 submission.py:307] 1500) loss = 4.585, grad_norm = 0.500 +I0913 09:57:58.087632 139773553383168 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.499999, loss=3.92981 +I0913 09:57:58.090636 139802006037696 submission.py:307] 2000) loss = 3.930, grad_norm = 0.500 +I0913 09:59:15.245643 139773544990464 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.499999, loss=3.42167 +I0913 09:59:15.248966 139802006037696 submission.py:307] 2500) loss = 3.422, grad_norm = 0.500 +I0913 10:00:32.398894 139773553383168 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.437466, loss=3.0317 +I0913 10:00:32.402305 139802006037696 submission.py:307] 3000) loss = 3.032, grad_norm = 0.437 +I0913 10:01:49.575073 139773544990464 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.398994, loss=2.88315 +I0913 10:01:49.578170 139802006037696 submission.py:307] 3500) loss = 2.883, grad_norm = 0.399 +I0913 10:03:06.732309 139773553383168 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.359962, loss=2.81904 +I0913 10:03:06.735445 139802006037696 submission.py:307] 4000) loss = 2.819, grad_norm = 0.360 +I0913 10:03:31.230466 139802006037696 spec.py:333] Evaluating on the training split. +I0913 10:03:33.429897 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 10:04:34.780100 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 10:04:36.962121 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 10:05:41.052552 139802006037696 spec.py:363] Evaluating on the test split. +I0913 10:05:43.228821 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 10:06:43.317482 139802006037696 submission_runner.py:516] Time since start: 1354.63s, Step: 4156, {'train/accuracy': 0.5489787078830985, 'train/loss': 2.59488128618976, 'train/bleu': 25.510182877108722, 'validation/accuracy': 0.5528263753704232, 'validation/loss': 2.5477869152273374, 'validation/bleu': 20.77749199731472, 'validation/num_examples': 3000, 'test/accuracy': 0.5523676718377781, 'test/loss': 2.5487264685375632, 'test/bleu': 19.208453271442007, 'test/num_examples': 3003, 'score': 718.3431355953217, 'total_duration': 1354.6345691680908, 'accumulated_submission_time': 718.3431355953217, 'accumulated_eval_time': 633.0395290851593, 'accumulated_logging_time': 0.034538984298706055} +I0913 10:06:43.343036 139773544990464 logging_writer.py:48] [4156] accumulated_eval_time=633.04, accumulated_logging_time=0.034539, accumulated_submission_time=718.343, global_step=4156, preemption_count=0, score=718.343, test/accuracy=0.552368, test/bleu=19.2085, test/loss=2.54873, test/num_examples=3003, total_duration=1354.63, train/accuracy=0.548979, train/bleu=25.5102, train/loss=2.59488, validation/accuracy=0.552826, validation/bleu=20.7775, validation/loss=2.54779, validation/num_examples=3000 +I0913 10:07:37.107796 139773553383168 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.344598, loss=2.63444 +I0913 10:07:37.110768 139802006037696 submission.py:307] 4500) loss = 2.634, grad_norm = 0.345 +I0913 10:08:54.278817 139773544990464 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.323757, loss=2.48891 +I0913 10:08:54.282252 139802006037696 submission.py:307] 5000) loss = 2.489, grad_norm = 0.324 +I0913 10:10:11.476021 139773553383168 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.320416, loss=2.52553 +I0913 10:10:11.479217 139802006037696 submission.py:307] 5500) loss = 2.526, grad_norm = 0.320 +I0913 10:11:28.683497 139773544990464 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.296125, loss=2.44763 +I0913 10:11:28.686748 139802006037696 submission.py:307] 6000) loss = 2.448, grad_norm = 0.296 +I0913 10:12:45.909964 139773553383168 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.301734, loss=2.31653 +I0913 10:12:45.913098 139802006037696 submission.py:307] 6500) loss = 2.317, grad_norm = 0.302 +I0913 10:14:03.180282 139773544990464 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.27677, loss=2.37129 +I0913 10:14:03.183641 139802006037696 submission.py:307] 7000) loss = 2.371, grad_norm = 0.277 +I0913 10:15:20.410509 139773553383168 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.261179, loss=2.23051 +I0913 10:15:20.413671 139802006037696 submission.py:307] 7500) loss = 2.231, grad_norm = 0.261 +I0913 10:16:37.633719 139773544990464 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.247378, loss=2.26924 +I0913 10:16:37.636728 139802006037696 submission.py:307] 8000) loss = 2.269, grad_norm = 0.247 +I0913 10:17:27.908026 139802006037696 spec.py:333] Evaluating on the training split. +I0913 10:17:30.109240 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 10:18:53.695098 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 10:18:55.872769 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 10:19:59.894174 139802006037696 spec.py:363] Evaluating on the test split. +I0913 10:20:02.077692 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 10:21:13.309611 139802006037696 submission_runner.py:516] Time since start: 2224.63s, Step: 8323, {'train/accuracy': 0.599668911976253, 'train/loss': 2.1029416171937436, 'train/bleu': 29.484997493987493, 'validation/accuracy': 0.610816976850876, 'validation/loss': 2.0018062004810853, 'validation/bleu': 24.84724125678546, 'validation/num_examples': 3000, 'test/accuracy': 0.6170937191331125, 'test/loss': 1.9664649424786473, 'test/bleu': 23.751160079640105, 'test/num_examples': 3003, 'score': 1360.511590719223, 'total_duration': 2224.6266939640045, 'accumulated_submission_time': 1360.511590719223, 'accumulated_eval_time': 858.4411356449127, 'accumulated_logging_time': 0.07004904747009277} +I0913 10:21:13.336900 139773553383168 logging_writer.py:48] [8323] accumulated_eval_time=858.441, accumulated_logging_time=0.070049, accumulated_submission_time=1360.51, global_step=8323, preemption_count=0, score=1360.51, test/accuracy=0.617094, test/bleu=23.7512, test/loss=1.96646, test/num_examples=3003, total_duration=2224.63, train/accuracy=0.599669, train/bleu=29.485, train/loss=2.10294, validation/accuracy=0.610817, validation/bleu=24.8472, validation/loss=2.00181, validation/num_examples=3000 +I0913 10:21:41.343808 139773544990464 logging_writer.py:48] [8500] global_step=8500, grad_norm=0.264622, loss=2.25805 +I0913 10:21:41.346896 139802006037696 submission.py:307] 8500) loss = 2.258, grad_norm = 0.265 +I0913 10:22:58.394151 139773553383168 logging_writer.py:48] [9000] global_step=9000, grad_norm=0.243149, loss=2.09102 +I0913 10:22:58.397281 139802006037696 submission.py:307] 9000) loss = 2.091, grad_norm = 0.243 +I0913 10:24:15.489920 139773544990464 logging_writer.py:48] [9500] global_step=9500, grad_norm=0.236884, loss=2.09534 +I0913 10:24:15.493361 139802006037696 submission.py:307] 9500) loss = 2.095, grad_norm = 0.237 +I0913 10:25:32.669098 139773553383168 logging_writer.py:48] [10000] global_step=10000, grad_norm=0.225057, loss=2.13352 +I0913 10:25:32.672359 139802006037696 submission.py:307] 10000) loss = 2.134, grad_norm = 0.225 +I0913 10:26:49.845029 139773544990464 logging_writer.py:48] [10500] global_step=10500, grad_norm=0.213144, loss=2.06226 +I0913 10:26:49.848198 139802006037696 submission.py:307] 10500) loss = 2.062, grad_norm = 0.213 +I0913 10:28:07.048337 139773553383168 logging_writer.py:48] [11000] global_step=11000, grad_norm=0.217083, loss=2.14075 +I0913 10:28:07.051543 139802006037696 submission.py:307] 11000) loss = 2.141, grad_norm = 0.217 +I0913 10:29:24.256427 139773544990464 logging_writer.py:48] [11500] global_step=11500, grad_norm=0.199603, loss=2.12959 +I0913 10:29:24.259827 139802006037696 submission.py:307] 11500) loss = 2.130, grad_norm = 0.200 +I0913 10:30:41.423550 139773553383168 logging_writer.py:48] [12000] global_step=12000, grad_norm=0.198029, loss=2.11394 +I0913 10:30:41.426659 139802006037696 submission.py:307] 12000) loss = 2.114, grad_norm = 0.198 +I0913 10:31:57.916598 139802006037696 spec.py:333] Evaluating on the training split. +I0913 10:32:00.115298 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 10:33:01.155299 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 10:33:03.340551 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 10:34:03.508654 139802006037696 spec.py:363] Evaluating on the test split. +I0913 10:34:05.691888 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 10:35:01.012098 139802006037696 submission_runner.py:516] Time since start: 3052.33s, Step: 12493, {'train/accuracy': 0.6162815929830172, 'train/loss': 1.9544963967153577, 'train/bleu': 29.869366276144724, 'validation/accuracy': 0.6350696209594425, 'validation/loss': 1.8175107872190053, 'validation/bleu': 26.66075888044792, 'validation/num_examples': 3000, 'test/accuracy': 0.6414037534135146, 'test/loss': 1.7672175498227878, 'test/bleu': 25.611929074622587, 'test/num_examples': 3003, 'score': 2002.6904723644257, 'total_duration': 3052.3292145729065, 'accumulated_submission_time': 2002.6904723644257, 'accumulated_eval_time': 1041.5366611480713, 'accumulated_logging_time': 0.10674023628234863} +I0913 10:35:01.037026 139773544990464 logging_writer.py:48] [12493] accumulated_eval_time=1041.54, accumulated_logging_time=0.10674, accumulated_submission_time=2002.69, global_step=12493, preemption_count=0, score=2002.69, test/accuracy=0.641404, test/bleu=25.6119, test/loss=1.76722, test/num_examples=3003, total_duration=3052.33, train/accuracy=0.616282, train/bleu=29.8694, train/loss=1.9545, validation/accuracy=0.63507, validation/bleu=26.6608, validation/loss=1.81751, validation/num_examples=3000 +I0913 10:35:02.906990 139773553383168 logging_writer.py:48] [12500] global_step=12500, grad_norm=0.194674, loss=2.0295 +I0913 10:35:02.910052 139802006037696 submission.py:307] 12500) loss = 2.029, grad_norm = 0.195 +I0913 10:36:19.797260 139773544990464 logging_writer.py:48] [13000] global_step=13000, grad_norm=0.193838, loss=2.0262 +I0913 10:36:19.801401 139802006037696 submission.py:307] 13000) loss = 2.026, grad_norm = 0.194 +I0913 10:37:36.851883 139773553383168 logging_writer.py:48] [13500] global_step=13500, grad_norm=0.187582, loss=2.14926 +I0913 10:37:36.854979 139802006037696 submission.py:307] 13500) loss = 2.149, grad_norm = 0.188 +I0913 10:38:54.003748 139773544990464 logging_writer.py:48] [14000] global_step=14000, grad_norm=0.180846, loss=2.02707 +I0913 10:38:54.006877 139802006037696 submission.py:307] 14000) loss = 2.027, grad_norm = 0.181 +I0913 10:40:11.098086 139773553383168 logging_writer.py:48] [14500] global_step=14500, grad_norm=0.185195, loss=2.08159 +I0913 10:40:11.101354 139802006037696 submission.py:307] 14500) loss = 2.082, grad_norm = 0.185 +I0913 10:41:28.177737 139773544990464 logging_writer.py:48] [15000] global_step=15000, grad_norm=0.186962, loss=2.07685 +I0913 10:41:28.181089 139802006037696 submission.py:307] 15000) loss = 2.077, grad_norm = 0.187 +I0913 10:42:45.263144 139773553383168 logging_writer.py:48] [15500] global_step=15500, grad_norm=0.177638, loss=2.12169 +I0913 10:42:45.266397 139802006037696 submission.py:307] 15500) loss = 2.122, grad_norm = 0.178 +I0913 10:44:02.337451 139773544990464 logging_writer.py:48] [16000] global_step=16000, grad_norm=0.184067, loss=1.97493 +I0913 10:44:02.340769 139802006037696 submission.py:307] 16000) loss = 1.975, grad_norm = 0.184 +I0913 10:45:19.421108 139773553383168 logging_writer.py:48] [16500] global_step=16500, grad_norm=0.174213, loss=1.97136 +I0913 10:45:19.424238 139802006037696 submission.py:307] 16500) loss = 1.971, grad_norm = 0.174 +I0913 10:45:45.723565 139802006037696 spec.py:333] Evaluating on the training split. +I0913 10:45:47.920369 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 10:46:51.543034 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 10:46:53.725222 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 10:48:00.500822 139802006037696 spec.py:363] Evaluating on the test split. +I0913 10:48:02.685475 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 10:49:01.089152 139802006037696 submission_runner.py:516] Time since start: 3892.41s, Step: 16668, {'train/accuracy': 0.6284328084950568, 'train/loss': 1.8549527559730867, 'train/bleu': 30.84180050089463, 'validation/accuracy': 0.6470967501952859, 'validation/loss': 1.721576414117618, 'validation/bleu': 27.63565652379737, 'validation/num_examples': 3000, 'test/accuracy': 0.6559642089361455, 'test/loss': 1.6560678853640114, 'test/bleu': 26.4104526056849, 'test/num_examples': 3003, 'score': 2644.968838453293, 'total_duration': 3892.406261205673, 'accumulated_submission_time': 2644.968838453293, 'accumulated_eval_time': 1236.902271747589, 'accumulated_logging_time': 0.14114046096801758} +I0913 10:49:01.114207 139773544990464 logging_writer.py:48] [16668] accumulated_eval_time=1236.9, accumulated_logging_time=0.14114, accumulated_submission_time=2644.97, global_step=16668, preemption_count=0, score=2644.97, test/accuracy=0.655964, test/bleu=26.4105, test/loss=1.65607, test/num_examples=3003, total_duration=3892.41, train/accuracy=0.628433, train/bleu=30.8418, train/loss=1.85495, validation/accuracy=0.647097, validation/bleu=27.6357, validation/loss=1.72158, validation/num_examples=3000 +I0913 10:49:52.895830 139773553383168 logging_writer.py:48] [17000] global_step=17000, grad_norm=0.169346, loss=1.96934 +I0913 10:49:52.899015 139802006037696 submission.py:307] 17000) loss = 1.969, grad_norm = 0.169 +I0913 10:51:09.864702 139773544990464 logging_writer.py:48] [17500] global_step=17500, grad_norm=0.163514, loss=1.9115 +I0913 10:51:09.867938 139802006037696 submission.py:307] 17500) loss = 1.912, grad_norm = 0.164 +I0913 10:52:26.867788 139773553383168 logging_writer.py:48] [18000] global_step=18000, grad_norm=0.164744, loss=1.90653 +I0913 10:52:26.870860 139802006037696 submission.py:307] 18000) loss = 1.907, grad_norm = 0.165 +I0913 10:53:43.914741 139773544990464 logging_writer.py:48] [18500] global_step=18500, grad_norm=0.159762, loss=1.91058 +I0913 10:53:43.917878 139802006037696 submission.py:307] 18500) loss = 1.911, grad_norm = 0.160 +I0913 10:55:00.979022 139773553383168 logging_writer.py:48] [19000] global_step=19000, grad_norm=0.173208, loss=1.8524 +I0913 10:55:00.982352 139802006037696 submission.py:307] 19000) loss = 1.852, grad_norm = 0.173 +I0913 10:56:18.021111 139773544990464 logging_writer.py:48] [19500] global_step=19500, grad_norm=0.152609, loss=1.83937 +I0913 10:56:18.024255 139802006037696 submission.py:307] 19500) loss = 1.839, grad_norm = 0.153 +I0913 10:57:35.094857 139773553383168 logging_writer.py:48] [20000] global_step=20000, grad_norm=0.168905, loss=1.93428 +I0913 10:57:35.098149 139802006037696 submission.py:307] 20000) loss = 1.934, grad_norm = 0.169 +I0913 10:58:52.224573 139773544990464 logging_writer.py:48] [20500] global_step=20500, grad_norm=0.155267, loss=1.8319 +I0913 10:58:52.228054 139802006037696 submission.py:307] 20500) loss = 1.832, grad_norm = 0.155 +I0913 10:59:45.797967 139802006037696 spec.py:333] Evaluating on the training split. +I0913 10:59:47.996783 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 11:01:12.089937 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 11:01:14.267692 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 11:02:17.986584 139802006037696 spec.py:363] Evaluating on the test split. +I0913 11:02:20.163747 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 11:03:16.739674 139802006037696 submission_runner.py:516] Time since start: 4748.06s, Step: 20845, {'train/accuracy': 0.6389054066416613, 'train/loss': 1.7704603735934499, 'train/bleu': 31.288025568108456, 'validation/accuracy': 0.6537922654399821, 'validation/loss': 1.6578037625075945, 'validation/bleu': 27.8906182266799, 'validation/num_examples': 3000, 'test/accuracy': 0.6640055778281332, 'test/loss': 1.5898763871942363, 'test/bleu': 27.026689310797952, 'test/num_examples': 3003, 'score': 3287.2435598373413, 'total_duration': 4748.056781768799, 'accumulated_submission_time': 3287.2435598373413, 'accumulated_eval_time': 1447.8440327644348, 'accumulated_logging_time': 0.17554354667663574} +I0913 11:03:16.766180 139773553383168 logging_writer.py:48] [20845] accumulated_eval_time=1447.84, accumulated_logging_time=0.175544, accumulated_submission_time=3287.24, global_step=20845, preemption_count=0, score=3287.24, test/accuracy=0.664006, test/bleu=27.0267, test/loss=1.58988, test/num_examples=3003, total_duration=4748.06, train/accuracy=0.638905, train/bleu=31.288, train/loss=1.77046, validation/accuracy=0.653792, validation/bleu=27.8906, validation/loss=1.6578, validation/num_examples=3000 +I0913 11:03:41.383091 139773544990464 logging_writer.py:48] [21000] global_step=21000, grad_norm=0.164389, loss=1.87994 +I0913 11:03:41.386016 139802006037696 submission.py:307] 21000) loss = 1.880, grad_norm = 0.164 +I0913 11:04:58.321394 139773553383168 logging_writer.py:48] [21500] global_step=21500, grad_norm=0.147912, loss=1.91022 +I0913 11:04:58.324454 139802006037696 submission.py:307] 21500) loss = 1.910, grad_norm = 0.148 +I0913 11:06:15.371547 139773544990464 logging_writer.py:48] [22000] global_step=22000, grad_norm=0.186467, loss=1.87206 +I0913 11:06:15.374780 139802006037696 submission.py:307] 22000) loss = 1.872, grad_norm = 0.186 +I0913 11:07:32.476555 139773553383168 logging_writer.py:48] [22500] global_step=22500, grad_norm=0.143874, loss=1.8264 +I0913 11:07:32.479753 139802006037696 submission.py:307] 22500) loss = 1.826, grad_norm = 0.144 +I0913 11:08:49.591516 139773544990464 logging_writer.py:48] [23000] global_step=23000, grad_norm=0.194025, loss=1.86313 +I0913 11:08:49.594768 139802006037696 submission.py:307] 23000) loss = 1.863, grad_norm = 0.194 +I0913 11:10:06.693825 139773553383168 logging_writer.py:48] [23500] global_step=23500, grad_norm=0.184061, loss=1.91842 +I0913 11:10:06.697023 139802006037696 submission.py:307] 23500) loss = 1.918, grad_norm = 0.184 +I0913 11:11:23.779703 139773544990464 logging_writer.py:48] [24000] global_step=24000, grad_norm=0.142878, loss=1.80651 +I0913 11:11:23.782911 139802006037696 submission.py:307] 24000) loss = 1.807, grad_norm = 0.143 +I0913 11:12:40.893857 139773553383168 logging_writer.py:48] [24500] global_step=24500, grad_norm=0.163096, loss=1.87674 +I0913 11:12:40.897163 139802006037696 submission.py:307] 24500) loss = 1.877, grad_norm = 0.163 +I0913 11:13:58.008685 139773544990464 logging_writer.py:48] [25000] global_step=25000, grad_norm=0.142597, loss=1.77854 +I0913 11:13:58.011802 139802006037696 submission.py:307] 25000) loss = 1.779, grad_norm = 0.143 +I0913 11:14:01.361281 139802006037696 spec.py:333] Evaluating on the training split. +I0913 11:14:03.563064 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 11:15:24.094252 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 11:15:26.275802 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 11:16:23.052695 139802006037696 spec.py:363] Evaluating on the test split. +I0913 11:16:25.232973 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 11:17:19.936973 139802006037696 submission_runner.py:516] Time since start: 5591.25s, Step: 25019, {'train/accuracy': 0.6479801763383853, 'train/loss': 1.6988456606926756, 'train/bleu': 32.448172318625886, 'validation/accuracy': 0.6600042156947837, 'validation/loss': 1.6171265700363293, 'validation/bleu': 28.196767541148606, 'validation/num_examples': 3000, 'test/accuracy': 0.6709546220440415, 'test/loss': 1.5374762143396665, 'test/bleu': 27.231862769804216, 'test/num_examples': 3003, 'score': 3929.4358291625977, 'total_duration': 5591.25407743454, 'accumulated_submission_time': 3929.4358291625977, 'accumulated_eval_time': 1646.4197652339935, 'accumulated_logging_time': 0.21161365509033203} +I0913 11:17:19.962797 139773553383168 logging_writer.py:48] [25019] accumulated_eval_time=1646.42, accumulated_logging_time=0.211614, accumulated_submission_time=3929.44, global_step=25019, preemption_count=0, score=3929.44, test/accuracy=0.670955, test/bleu=27.2319, test/loss=1.53748, test/num_examples=3003, total_duration=5591.25, train/accuracy=0.64798, train/bleu=32.4482, train/loss=1.69885, validation/accuracy=0.660004, validation/bleu=28.1968, validation/loss=1.61713, validation/num_examples=3000 +I0913 11:18:34.798184 139773544990464 logging_writer.py:48] [25500] global_step=25500, grad_norm=0.151862, loss=1.82077 +I0913 11:18:34.801446 139802006037696 submission.py:307] 25500) loss = 1.821, grad_norm = 0.152 +I0913 11:19:51.932808 139773553383168 logging_writer.py:48] [26000] global_step=26000, grad_norm=0.148824, loss=1.84454 +I0913 11:19:51.936017 139802006037696 submission.py:307] 26000) loss = 1.845, grad_norm = 0.149 +I0913 11:21:09.098613 139773544990464 logging_writer.py:48] [26500] global_step=26500, grad_norm=0.165738, loss=1.7781 +I0913 11:21:09.101641 139802006037696 submission.py:307] 26500) loss = 1.778, grad_norm = 0.166 +I0913 11:22:26.302416 139773553383168 logging_writer.py:48] [27000] global_step=27000, grad_norm=0.164001, loss=1.83294 +I0913 11:22:26.305701 139802006037696 submission.py:307] 27000) loss = 1.833, grad_norm = 0.164 +I0913 11:23:43.482774 139773544990464 logging_writer.py:48] [27500] global_step=27500, grad_norm=0.158346, loss=1.87063 +I0913 11:23:43.485952 139802006037696 submission.py:307] 27500) loss = 1.871, grad_norm = 0.158 +I0913 11:25:00.679435 139773553383168 logging_writer.py:48] [28000] global_step=28000, grad_norm=0.155821, loss=1.88188 +I0913 11:25:00.682569 139802006037696 submission.py:307] 28000) loss = 1.882, grad_norm = 0.156 +I0913 11:26:17.879315 139773544990464 logging_writer.py:48] [28500] global_step=28500, grad_norm=0.151122, loss=1.78291 +I0913 11:26:17.882511 139802006037696 submission.py:307] 28500) loss = 1.783, grad_norm = 0.151 +I0913 11:27:35.085067 139773553383168 logging_writer.py:48] [29000] global_step=29000, grad_norm=0.154204, loss=1.77676 +I0913 11:27:35.088270 139802006037696 submission.py:307] 29000) loss = 1.777, grad_norm = 0.154 +I0913 11:28:04.565356 139802006037696 spec.py:333] Evaluating on the training split. +I0913 11:28:06.765980 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 11:29:21.347280 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 11:29:23.531570 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 11:30:31.332303 139802006037696 spec.py:363] Evaluating on the test split. +I0913 11:30:33.515067 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 11:31:44.064798 139802006037696 submission_runner.py:516] Time since start: 6455.38s, Step: 29188, {'train/accuracy': 0.6450799086757991, 'train/loss': 1.7099643264840183, 'train/bleu': 32.059144285066935, 'validation/accuracy': 0.6662409641541952, 'validation/loss': 1.5777422745843201, 'validation/bleu': 28.76029220781709, 'validation/num_examples': 3000, 'test/accuracy': 0.6765092092266574, 'test/loss': 1.500034679855906, 'test/bleu': 27.98602990649897, 'test/num_examples': 3003, 'score': 4571.648406744003, 'total_duration': 6455.38191485405, 'accumulated_submission_time': 4571.648406744003, 'accumulated_eval_time': 1865.9192254543304, 'accumulated_logging_time': 0.24688220024108887} +I0913 11:31:44.091316 139773544990464 logging_writer.py:48] [29188] accumulated_eval_time=1865.92, accumulated_logging_time=0.246882, accumulated_submission_time=4571.65, global_step=29188, preemption_count=0, score=4571.65, test/accuracy=0.676509, test/bleu=27.986, test/loss=1.50003, test/num_examples=3003, total_duration=6455.38, train/accuracy=0.64508, train/bleu=32.0591, train/loss=1.70996, validation/accuracy=0.666241, validation/bleu=28.7603, validation/loss=1.57774, validation/num_examples=3000 +I0913 11:32:32.874992 139773553383168 logging_writer.py:48] [29500] global_step=29500, grad_norm=0.200519, loss=1.80087 +I0913 11:32:32.878125 139802006037696 submission.py:307] 29500) loss = 1.801, grad_norm = 0.201 +I0913 11:33:49.974752 139773544990464 logging_writer.py:48] [30000] global_step=30000, grad_norm=0.156264, loss=1.84587 +I0913 11:33:49.978169 139802006037696 submission.py:307] 30000) loss = 1.846, grad_norm = 0.156 +I0913 11:35:07.149222 139773553383168 logging_writer.py:48] [30500] global_step=30500, grad_norm=0.150692, loss=1.80095 +I0913 11:35:07.152397 139802006037696 submission.py:307] 30500) loss = 1.801, grad_norm = 0.151 +I0913 11:36:24.325364 139773544990464 logging_writer.py:48] [31000] global_step=31000, grad_norm=0.154066, loss=1.6852 +I0913 11:36:24.328627 139802006037696 submission.py:307] 31000) loss = 1.685, grad_norm = 0.154 +I0913 11:37:41.455291 139773553383168 logging_writer.py:48] [31500] global_step=31500, grad_norm=0.182457, loss=1.74839 +I0913 11:37:41.458469 139802006037696 submission.py:307] 31500) loss = 1.748, grad_norm = 0.182 +I0913 11:38:58.595113 139773544990464 logging_writer.py:48] [32000] global_step=32000, grad_norm=0.152771, loss=1.69751 +I0913 11:38:58.598673 139802006037696 submission.py:307] 32000) loss = 1.698, grad_norm = 0.153 +I0913 11:40:15.675140 139773553383168 logging_writer.py:48] [32500] global_step=32500, grad_norm=0.183788, loss=1.77311 +I0913 11:40:15.678424 139802006037696 submission.py:307] 32500) loss = 1.773, grad_norm = 0.184 +I0913 11:41:32.752123 139773544990464 logging_writer.py:48] [33000] global_step=33000, grad_norm=0.164142, loss=1.77293 +I0913 11:41:32.755241 139802006037696 submission.py:307] 33000) loss = 1.773, grad_norm = 0.164 +I0913 11:42:28.643548 139802006037696 spec.py:333] Evaluating on the training split. +I0913 11:42:30.838177 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 11:44:06.003201 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 11:44:08.177913 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 11:45:22.460049 139802006037696 spec.py:363] Evaluating on the test split. +I0913 11:45:24.644288 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 11:46:41.786948 139802006037696 submission_runner.py:516] Time since start: 7353.10s, Step: 33360, {'train/accuracy': 0.6554331141606909, 'train/loss': 1.6377410206992142, 'train/bleu': 32.579644058317534, 'validation/accuracy': 0.6679768384768943, 'validation/loss': 1.5531466209346443, 'validation/bleu': 29.096854810739405, 'validation/num_examples': 3000, 'test/accuracy': 0.6807390622276451, 'test/loss': 1.468950725553425, 'test/bleu': 28.493779819136062, 'test/num_examples': 3003, 'score': 5213.796993970871, 'total_duration': 7353.1040625572205, 'accumulated_submission_time': 5213.796993970871, 'accumulated_eval_time': 2119.0626318454742, 'accumulated_logging_time': 0.28296875953674316} +I0913 11:46:41.813963 139773553383168 logging_writer.py:48] [33360] accumulated_eval_time=2119.06, accumulated_logging_time=0.282969, accumulated_submission_time=5213.8, global_step=33360, preemption_count=0, score=5213.8, test/accuracy=0.680739, test/bleu=28.4938, test/loss=1.46895, test/num_examples=3003, total_duration=7353.1, train/accuracy=0.655433, train/bleu=32.5796, train/loss=1.63774, validation/accuracy=0.667977, validation/bleu=29.0969, validation/loss=1.55315, validation/num_examples=3000 +I0913 11:47:04.105608 139773544990464 logging_writer.py:48] [33500] global_step=33500, grad_norm=0.153303, loss=1.7106 +I0913 11:47:04.108906 139802006037696 submission.py:307] 33500) loss = 1.711, grad_norm = 0.153 +I0913 11:48:21.090594 139773553383168 logging_writer.py:48] [34000] global_step=34000, grad_norm=0.163155, loss=1.76048 +I0913 11:48:21.093952 139802006037696 submission.py:307] 34000) loss = 1.760, grad_norm = 0.163 +I0913 11:49:38.178753 139773544990464 logging_writer.py:48] [34500] global_step=34500, grad_norm=0.186466, loss=1.83457 +I0913 11:49:38.181876 139802006037696 submission.py:307] 34500) loss = 1.835, grad_norm = 0.186 +I0913 11:50:55.299475 139773553383168 logging_writer.py:48] [35000] global_step=35000, grad_norm=0.17412, loss=1.70063 +I0913 11:50:55.302663 139802006037696 submission.py:307] 35000) loss = 1.701, grad_norm = 0.174 +I0913 11:52:12.408060 139773544990464 logging_writer.py:48] [35500] global_step=35500, grad_norm=0.18763, loss=1.69651 +I0913 11:52:12.411411 139802006037696 submission.py:307] 35500) loss = 1.697, grad_norm = 0.188 +I0913 11:53:29.515783 139773553383168 logging_writer.py:48] [36000] global_step=36000, grad_norm=0.195526, loss=1.69882 +I0913 11:53:29.518977 139802006037696 submission.py:307] 36000) loss = 1.699, grad_norm = 0.196 +I0913 11:54:46.645999 139773544990464 logging_writer.py:48] [36500] global_step=36500, grad_norm=0.182566, loss=1.78823 +I0913 11:54:46.649212 139802006037696 submission.py:307] 36500) loss = 1.788, grad_norm = 0.183 +I0913 11:56:03.771008 139773553383168 logging_writer.py:48] [37000] global_step=37000, grad_norm=0.238375, loss=1.70364 +I0913 11:56:03.774312 139802006037696 submission.py:307] 37000) loss = 1.704, grad_norm = 0.238 +I0913 11:57:20.884901 139773544990464 logging_writer.py:48] [37500] global_step=37500, grad_norm=0.20865, loss=1.79938 +I0913 11:57:20.888087 139802006037696 submission.py:307] 37500) loss = 1.799, grad_norm = 0.209 +I0913 11:57:26.392853 139802006037696 spec.py:333] Evaluating on the training split. +I0913 11:57:28.591342 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 11:58:45.242381 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 11:58:47.418229 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 11:59:47.223197 139802006037696 spec.py:363] Evaluating on the test split. +I0913 11:59:49.402974 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 12:00:48.374267 139802006037696 submission_runner.py:516] Time since start: 8199.69s, Step: 37533, {'train/accuracy': 0.6648527895520678, 'train/loss': 1.578003637300951, 'train/bleu': 33.55789683123363, 'validation/accuracy': 0.6717337664753071, 'validation/loss': 1.5327072819927838, 'validation/bleu': 29.151142586857883, 'validation/num_examples': 3000, 'test/accuracy': 0.6851664633083493, 'test/loss': 1.4453388276393004, 'test/bleu': 28.743940528054015, 'test/num_examples': 3003, 'score': 5855.9824957847595, 'total_duration': 8199.691377401352, 'accumulated_submission_time': 5855.9824957847595, 'accumulated_eval_time': 2321.044037103653, 'accumulated_logging_time': 0.3195197582244873} +I0913 12:00:48.402305 139773553383168 logging_writer.py:48] [37533] accumulated_eval_time=2321.04, accumulated_logging_time=0.31952, accumulated_submission_time=5855.98, global_step=37533, preemption_count=0, score=5855.98, test/accuracy=0.685166, test/bleu=28.7439, test/loss=1.44534, test/num_examples=3003, total_duration=8199.69, train/accuracy=0.664853, train/bleu=33.5579, train/loss=1.578, validation/accuracy=0.671734, validation/bleu=29.1511, validation/loss=1.53271, validation/num_examples=3000 +I0913 12:02:00.931043 139773544990464 logging_writer.py:48] [38000] global_step=38000, grad_norm=0.231498, loss=1.64566 +I0913 12:02:00.937463 139802006037696 submission.py:307] 38000) loss = 1.646, grad_norm = 0.231 +I0913 12:03:17.946426 139773553383168 logging_writer.py:48] [38500] global_step=38500, grad_norm=0.183864, loss=1.69031 +I0913 12:03:17.949676 139802006037696 submission.py:307] 38500) loss = 1.690, grad_norm = 0.184 +I0913 12:04:35.065276 139773544990464 logging_writer.py:48] [39000] global_step=39000, grad_norm=0.204933, loss=1.76605 +I0913 12:04:35.068700 139802006037696 submission.py:307] 39000) loss = 1.766, grad_norm = 0.205 +I0913 12:05:52.190950 139773553383168 logging_writer.py:48] [39500] global_step=39500, grad_norm=0.184146, loss=1.72953 +I0913 12:05:52.194152 139802006037696 submission.py:307] 39500) loss = 1.730, grad_norm = 0.184 +I0913 12:07:09.344909 139773544990464 logging_writer.py:48] [40000] global_step=40000, grad_norm=0.208821, loss=1.67559 +I0913 12:07:09.348014 139802006037696 submission.py:307] 40000) loss = 1.676, grad_norm = 0.209 +I0913 12:08:26.517569 139773553383168 logging_writer.py:48] [40500] global_step=40500, grad_norm=0.229631, loss=1.71278 +I0913 12:08:26.520751 139802006037696 submission.py:307] 40500) loss = 1.713, grad_norm = 0.230 +I0913 12:09:43.635562 139773544990464 logging_writer.py:48] [41000] global_step=41000, grad_norm=0.246328, loss=1.73237 +I0913 12:09:43.638957 139802006037696 submission.py:307] 41000) loss = 1.732, grad_norm = 0.246 +I0913 12:11:00.797864 139773553383168 logging_writer.py:48] [41500] global_step=41500, grad_norm=0.25929, loss=1.6646 +I0913 12:11:00.801008 139802006037696 submission.py:307] 41500) loss = 1.665, grad_norm = 0.259 +I0913 12:11:32.995007 139802006037696 spec.py:333] Evaluating on the training split. +I0913 12:11:35.192687 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 12:13:01.060266 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 12:13:03.241083 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 12:14:14.428789 139802006037696 spec.py:363] Evaluating on the test split. +I0913 12:14:16.612259 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 12:15:28.732513 139802006037696 submission_runner.py:516] Time since start: 9080.05s, Step: 41706, {'train/accuracy': 0.6605701557997221, 'train/loss': 1.5903092314492704, 'train/bleu': 32.826952634063396, 'validation/accuracy': 0.673891210276376, 'validation/loss': 1.5142536399114703, 'validation/bleu': 29.618207209392207, 'validation/num_examples': 3000, 'test/accuracy': 0.686456336064145, 'test/loss': 1.4227539800127824, 'test/bleu': 28.718436773021423, 'test/num_examples': 3003, 'score': 6498.172658920288, 'total_duration': 9080.049595355988, 'accumulated_submission_time': 6498.172658920288, 'accumulated_eval_time': 2556.7816529273987, 'accumulated_logging_time': 0.3569211959838867} +I0913 12:15:28.759973 139773544990464 logging_writer.py:48] [41706] accumulated_eval_time=2556.78, accumulated_logging_time=0.356921, accumulated_submission_time=6498.17, global_step=41706, preemption_count=0, score=6498.17, test/accuracy=0.686456, test/bleu=28.7184, test/loss=1.42275, test/num_examples=3003, total_duration=9080.05, train/accuracy=0.66057, train/bleu=32.827, train/loss=1.59031, validation/accuracy=0.673891, validation/bleu=29.6182, validation/loss=1.51425, validation/num_examples=3000 +I0913 12:16:14.738763 139773553383168 logging_writer.py:48] [42000] global_step=42000, grad_norm=0.199043, loss=1.61645 +I0913 12:16:14.741991 139802006037696 submission.py:307] 42000) loss = 1.616, grad_norm = 0.199 +I0913 12:17:31.701560 139773544990464 logging_writer.py:48] [42500] global_step=42500, grad_norm=0.218142, loss=1.71347 +I0913 12:17:31.704827 139802006037696 submission.py:307] 42500) loss = 1.713, grad_norm = 0.218 +I0913 12:18:48.806424 139773553383168 logging_writer.py:48] [43000] global_step=43000, grad_norm=0.206222, loss=1.62188 +I0913 12:18:48.809688 139802006037696 submission.py:307] 43000) loss = 1.622, grad_norm = 0.206 +I0913 12:20:05.873318 139773544990464 logging_writer.py:48] [43500] global_step=43500, grad_norm=0.207112, loss=1.67911 +I0913 12:20:05.876397 139802006037696 submission.py:307] 43500) loss = 1.679, grad_norm = 0.207 +I0913 12:21:22.966707 139773553383168 logging_writer.py:48] [44000] global_step=44000, grad_norm=0.226646, loss=1.73414 +I0913 12:21:22.969921 139802006037696 submission.py:307] 44000) loss = 1.734, grad_norm = 0.227 +I0913 12:22:40.042870 139773544990464 logging_writer.py:48] [44500] global_step=44500, grad_norm=0.246891, loss=1.76099 +I0913 12:22:40.046038 139802006037696 submission.py:307] 44500) loss = 1.761, grad_norm = 0.247 +I0913 12:23:57.135301 139773553383168 logging_writer.py:48] [45000] global_step=45000, grad_norm=0.239375, loss=1.60755 +I0913 12:23:57.138653 139802006037696 submission.py:307] 45000) loss = 1.608, grad_norm = 0.239 +I0913 12:25:14.246610 139773544990464 logging_writer.py:48] [45500] global_step=45500, grad_norm=0.244926, loss=1.68635 +I0913 12:25:14.249860 139802006037696 submission.py:307] 45500) loss = 1.686, grad_norm = 0.245 +I0913 12:26:13.393885 139802006037696 spec.py:333] Evaluating on the training split. +I0913 12:26:15.588795 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 12:27:38.795141 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 12:27:40.972972 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 12:28:47.565044 139802006037696 spec.py:363] Evaluating on the test split. +I0913 12:28:49.742521 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 12:30:06.812841 139802006037696 submission_runner.py:516] Time since start: 9958.13s, Step: 45881, {'train/accuracy': 0.6687904705841615, 'train/loss': 1.54301550578852, 'train/bleu': 33.26489292961415, 'validation/accuracy': 0.6759122639520899, 'validation/loss': 1.4998456892970948, 'validation/bleu': 29.255674651771734, 'validation/num_examples': 3000, 'test/accuracy': 0.6897333100923828, 'test/loss': 1.4008395793387949, 'test/bleu': 29.01833424750774, 'test/num_examples': 3003, 'score': 7140.401864767075, 'total_duration': 9958.129945755005, 'accumulated_submission_time': 7140.401864767075, 'accumulated_eval_time': 2790.200603723526, 'accumulated_logging_time': 0.3936951160430908} +I0913 12:30:06.841387 139773553383168 logging_writer.py:48] [45881] accumulated_eval_time=2790.2, accumulated_logging_time=0.393695, accumulated_submission_time=7140.4, global_step=45881, preemption_count=0, score=7140.4, test/accuracy=0.689733, test/bleu=29.0183, test/loss=1.40084, test/num_examples=3003, total_duration=9958.13, train/accuracy=0.66879, train/bleu=33.2649, train/loss=1.54302, validation/accuracy=0.675912, validation/bleu=29.2557, validation/loss=1.49985, validation/num_examples=3000 +I0913 12:30:25.955293 139773544990464 logging_writer.py:48] [46000] global_step=46000, grad_norm=0.23122, loss=1.64799 +I0913 12:30:25.958334 139802006037696 submission.py:307] 46000) loss = 1.648, grad_norm = 0.231 +I0913 12:31:42.858620 139773553383168 logging_writer.py:48] [46500] global_step=46500, grad_norm=0.232899, loss=1.69746 +I0913 12:31:42.861910 139802006037696 submission.py:307] 46500) loss = 1.697, grad_norm = 0.233 +I0913 12:32:59.941899 139773544990464 logging_writer.py:48] [47000] global_step=47000, grad_norm=0.241357, loss=1.61268 +I0913 12:32:59.944977 139802006037696 submission.py:307] 47000) loss = 1.613, grad_norm = 0.241 +I0913 12:34:17.061096 139773553383168 logging_writer.py:48] [47500] global_step=47500, grad_norm=0.254648, loss=1.64053 +I0913 12:34:17.064172 139802006037696 submission.py:307] 47500) loss = 1.641, grad_norm = 0.255 +I0913 12:35:34.185307 139773544990464 logging_writer.py:48] [48000] global_step=48000, grad_norm=0.2219, loss=1.70004 +I0913 12:35:34.188471 139802006037696 submission.py:307] 48000) loss = 1.700, grad_norm = 0.222 +I0913 12:36:51.331729 139773553383168 logging_writer.py:48] [48500] global_step=48500, grad_norm=0.217267, loss=1.65047 +I0913 12:36:51.334984 139802006037696 submission.py:307] 48500) loss = 1.650, grad_norm = 0.217 +I0913 12:38:08.505762 139773544990464 logging_writer.py:48] [49000] global_step=49000, grad_norm=0.234926, loss=1.72375 +I0913 12:38:08.508776 139802006037696 submission.py:307] 49000) loss = 1.724, grad_norm = 0.235 +I0913 12:39:25.627668 139773553383168 logging_writer.py:48] [49500] global_step=49500, grad_norm=0.252741, loss=1.69911 +I0913 12:39:25.631020 139802006037696 submission.py:307] 49500) loss = 1.699, grad_norm = 0.253 +I0913 12:40:42.753154 139773544990464 logging_writer.py:48] [50000] global_step=50000, grad_norm=0.241565, loss=1.68965 +I0913 12:40:42.756366 139802006037696 submission.py:307] 50000) loss = 1.690, grad_norm = 0.242 +I0913 12:40:51.500576 139802006037696 spec.py:333] Evaluating on the training split. +I0913 12:40:53.700704 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 12:42:25.440168 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 12:42:27.613372 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 12:43:24.862338 139802006037696 spec.py:363] Evaluating on the test split. +I0913 12:43:27.042448 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 12:44:32.502264 139802006037696 submission_runner.py:516] Time since start: 10823.82s, Step: 50054, {'train/accuracy': 0.6735101619014813, 'train/loss': 1.5062322382018603, 'train/bleu': 33.436885790865766, 'validation/accuracy': 0.6768669948295744, 'validation/loss': 1.4853608255012336, 'validation/bleu': 29.795995027663388, 'validation/num_examples': 3000, 'test/accuracy': 0.6921503689500901, 'test/loss': 1.391680829120911, 'test/bleu': 29.1076151397941, 'test/num_examples': 3003, 'score': 7782.6305565834045, 'total_duration': 10823.819381952286, 'accumulated_submission_time': 7782.6305565834045, 'accumulated_eval_time': 3011.202365875244, 'accumulated_logging_time': 0.46454644203186035} +I0913 12:44:32.531146 139773553383168 logging_writer.py:48] [50054] accumulated_eval_time=3011.2, accumulated_logging_time=0.464546, accumulated_submission_time=7782.63, global_step=50054, preemption_count=0, score=7782.63, test/accuracy=0.69215, test/bleu=29.1076, test/loss=1.39168, test/num_examples=3003, total_duration=10823.8, train/accuracy=0.67351, train/bleu=33.4369, train/loss=1.50623, validation/accuracy=0.676867, validation/bleu=29.796, validation/loss=1.48536, validation/num_examples=3000 +I0913 12:45:41.818753 139773544990464 logging_writer.py:48] [50500] global_step=50500, grad_norm=0.229865, loss=1.59529 +I0913 12:45:41.821968 139802006037696 submission.py:307] 50500) loss = 1.595, grad_norm = 0.230 +I0913 12:46:58.836527 139773553383168 logging_writer.py:48] [51000] global_step=51000, grad_norm=0.246494, loss=1.5569 +I0913 12:46:58.839672 139802006037696 submission.py:307] 51000) loss = 1.557, grad_norm = 0.246 +I0913 12:48:15.975049 139773544990464 logging_writer.py:48] [51500] global_step=51500, grad_norm=0.229042, loss=1.61589 +I0913 12:48:15.978237 139802006037696 submission.py:307] 51500) loss = 1.616, grad_norm = 0.229 +I0913 12:49:33.129830 139773553383168 logging_writer.py:48] [52000] global_step=52000, grad_norm=0.267914, loss=1.58511 +I0913 12:49:33.133196 139802006037696 submission.py:307] 52000) loss = 1.585, grad_norm = 0.268 +I0913 12:50:50.235316 139773544990464 logging_writer.py:48] [52500] global_step=52500, grad_norm=0.280962, loss=1.71088 +I0913 12:50:50.238456 139802006037696 submission.py:307] 52500) loss = 1.711, grad_norm = 0.281 +I0913 12:52:07.360044 139773553383168 logging_writer.py:48] [53000] global_step=53000, grad_norm=0.26892, loss=1.67061 +I0913 12:52:07.363291 139802006037696 submission.py:307] 53000) loss = 1.671, grad_norm = 0.269 +I0913 12:53:24.523355 139773544990464 logging_writer.py:48] [53500] global_step=53500, grad_norm=0.282228, loss=1.68564 +I0913 12:53:24.526506 139802006037696 submission.py:307] 53500) loss = 1.686, grad_norm = 0.282 +I0913 12:54:41.659328 139773553383168 logging_writer.py:48] [54000] global_step=54000, grad_norm=0.257824, loss=1.6326 +I0913 12:54:41.662755 139802006037696 submission.py:307] 54000) loss = 1.633, grad_norm = 0.258 +I0913 12:55:17.230287 139802006037696 spec.py:333] Evaluating on the training split. +I0913 12:55:19.427311 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 12:56:40.072100 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 12:56:42.249994 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 12:57:44.993486 139802006037696 spec.py:363] Evaluating on the test split. +I0913 12:57:47.181949 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 12:58:49.424720 139802006037696 submission_runner.py:516] Time since start: 11680.74s, Step: 54228, {'train/accuracy': 0.6695961450206517, 'train/loss': 1.5323578763194126, 'train/bleu': 33.85869854742274, 'validation/accuracy': 0.6803139452703624, 'validation/loss': 1.4672586204758775, 'validation/bleu': 29.93200702476283, 'validation/num_examples': 3000, 'test/accuracy': 0.695775957236651, 'test/loss': 1.3669638966358724, 'test/bleu': 29.462941877924937, 'test/num_examples': 3003, 'score': 8424.923278331757, 'total_duration': 11680.741777420044, 'accumulated_submission_time': 8424.923278331757, 'accumulated_eval_time': 3223.3967576026917, 'accumulated_logging_time': 0.5028431415557861} +I0913 12:58:49.456344 139773544990464 logging_writer.py:48] [54228] accumulated_eval_time=3223.4, accumulated_logging_time=0.502843, accumulated_submission_time=8424.92, global_step=54228, preemption_count=0, score=8424.92, test/accuracy=0.695776, test/bleu=29.4629, test/loss=1.36696, test/num_examples=3003, total_duration=11680.7, train/accuracy=0.669596, train/bleu=33.8587, train/loss=1.53236, validation/accuracy=0.680314, validation/bleu=29.932, validation/loss=1.46726, validation/num_examples=3000 +I0913 12:59:31.992679 139773553383168 logging_writer.py:48] [54500] global_step=54500, grad_norm=0.289258, loss=1.61393 +I0913 12:59:31.995714 139802006037696 submission.py:307] 54500) loss = 1.614, grad_norm = 0.289 +I0913 13:00:48.892746 139773544990464 logging_writer.py:48] [55000] global_step=55000, grad_norm=0.271938, loss=1.61001 +I0913 13:00:48.896205 139802006037696 submission.py:307] 55000) loss = 1.610, grad_norm = 0.272 +I0913 13:02:05.921702 139773553383168 logging_writer.py:48] [55500] global_step=55500, grad_norm=0.284979, loss=1.54577 +I0913 13:02:05.925019 139802006037696 submission.py:307] 55500) loss = 1.546, grad_norm = 0.285 +I0913 13:03:23.015601 139773544990464 logging_writer.py:48] [56000] global_step=56000, grad_norm=0.269365, loss=1.60534 +I0913 13:03:23.018676 139802006037696 submission.py:307] 56000) loss = 1.605, grad_norm = 0.269 +I0913 13:04:40.100562 139773553383168 logging_writer.py:48] [56500] global_step=56500, grad_norm=0.282703, loss=1.60494 +I0913 13:04:40.103813 139802006037696 submission.py:307] 56500) loss = 1.605, grad_norm = 0.283 +I0913 13:05:57.167060 139773544990464 logging_writer.py:48] [57000] global_step=57000, grad_norm=0.296616, loss=1.51779 +I0913 13:05:57.170479 139802006037696 submission.py:307] 57000) loss = 1.518, grad_norm = 0.297 +I0913 13:07:14.240674 139773553383168 logging_writer.py:48] [57500] global_step=57500, grad_norm=0.3009, loss=1.65193 +I0913 13:07:14.243881 139802006037696 submission.py:307] 57500) loss = 1.652, grad_norm = 0.301 +I0913 13:08:31.341167 139773544990464 logging_writer.py:48] [58000] global_step=58000, grad_norm=0.347013, loss=1.61188 +I0913 13:08:31.344500 139802006037696 submission.py:307] 58000) loss = 1.612, grad_norm = 0.347 +I0913 13:09:34.015447 139802006037696 spec.py:333] Evaluating on the training split. +I0913 13:09:36.214050 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 13:11:03.954389 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 13:11:06.138240 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 13:12:14.422135 139802006037696 spec.py:363] Evaluating on the test split. +I0913 13:12:16.595487 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 13:13:18.000437 139802006037696 submission_runner.py:516] Time since start: 12549.32s, Step: 58404, {'train/accuracy': 0.675087410563364, 'train/loss': 1.495353600331118, 'train/bleu': 33.75690313913292, 'validation/accuracy': 0.6813926671708969, 'validation/loss': 1.4579864167834249, 'validation/bleu': 29.651606972963968, 'validation/num_examples': 3000, 'test/accuracy': 0.6966591133577363, 'test/loss': 1.3583626568764162, 'test/bleu': 29.3762478734997, 'test/num_examples': 3003, 'score': 9067.07328248024, 'total_duration': 12549.31753706932, 'accumulated_submission_time': 9067.07328248024, 'accumulated_eval_time': 3447.381803750992, 'accumulated_logging_time': 0.5438311100006104} +I0913 13:13:18.030721 139773553383168 logging_writer.py:48] [58404] accumulated_eval_time=3447.38, accumulated_logging_time=0.543831, accumulated_submission_time=9067.07, global_step=58404, preemption_count=0, score=9067.07, test/accuracy=0.696659, test/bleu=29.3762, test/loss=1.35836, test/num_examples=3003, total_duration=12549.3, train/accuracy=0.675087, train/bleu=33.7569, train/loss=1.49535, validation/accuracy=0.681393, validation/bleu=29.6516, validation/loss=1.45799, validation/num_examples=3000 +I0913 13:13:33.563869 139773544990464 logging_writer.py:48] [58500] global_step=58500, grad_norm=0.292493, loss=1.6101 +I0913 13:13:33.566994 139802006037696 submission.py:307] 58500) loss = 1.610, grad_norm = 0.292 +I0913 13:14:50.478273 139773553383168 logging_writer.py:48] [59000] global_step=59000, grad_norm=0.290375, loss=1.61852 +I0913 13:14:50.481456 139802006037696 submission.py:307] 59000) loss = 1.619, grad_norm = 0.290 +I0913 13:16:07.519208 139773544990464 logging_writer.py:48] [59500] global_step=59500, grad_norm=0.336627, loss=1.53261 +I0913 13:16:07.522742 139802006037696 submission.py:307] 59500) loss = 1.533, grad_norm = 0.337 +I0913 13:17:24.600672 139773553383168 logging_writer.py:48] [60000] global_step=60000, grad_norm=0.288505, loss=1.56202 +I0913 13:17:24.603927 139802006037696 submission.py:307] 60000) loss = 1.562, grad_norm = 0.289 +I0913 13:18:41.730637 139773544990464 logging_writer.py:48] [60500] global_step=60500, grad_norm=0.284534, loss=1.62529 +I0913 13:18:41.734025 139802006037696 submission.py:307] 60500) loss = 1.625, grad_norm = 0.285 +I0913 13:19:58.850672 139773553383168 logging_writer.py:48] [61000] global_step=61000, grad_norm=0.288286, loss=1.61491 +I0913 13:19:58.853868 139802006037696 submission.py:307] 61000) loss = 1.615, grad_norm = 0.288 +I0913 13:21:15.957087 139773544990464 logging_writer.py:48] [61500] global_step=61500, grad_norm=0.312885, loss=1.49368 +I0913 13:21:15.960423 139802006037696 submission.py:307] 61500) loss = 1.494, grad_norm = 0.313 +I0913 13:22:33.048507 139773553383168 logging_writer.py:48] [62000] global_step=62000, grad_norm=0.27702, loss=1.63234 +I0913 13:22:33.051714 139802006037696 submission.py:307] 62000) loss = 1.632, grad_norm = 0.277 +I0913 13:23:50.141917 139773544990464 logging_writer.py:48] [62500] global_step=62500, grad_norm=0.285192, loss=1.58598 +I0913 13:23:50.145145 139802006037696 submission.py:307] 62500) loss = 1.586, grad_norm = 0.285 +I0913 13:24:02.588609 139802006037696 spec.py:333] Evaluating on the training split. +I0913 13:24:04.788494 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 13:25:34.817617 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 13:25:36.992686 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 13:26:46.564428 139802006037696 spec.py:363] Evaluating on the test split. +I0913 13:26:48.746566 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 13:27:55.455781 139802006037696 submission_runner.py:516] Time since start: 13426.77s, Step: 62578, {'train/accuracy': 0.6781173649443489, 'train/loss': 1.473514575179848, 'train/bleu': 34.9030195664067, 'validation/accuracy': 0.6826573756060061, 'validation/loss': 1.4489792439027414, 'validation/bleu': 30.034928923864545, 'validation/num_examples': 3000, 'test/accuracy': 0.6990645517401661, 'test/loss': 1.3470577501307304, 'test/bleu': 29.629351525062226, 'test/num_examples': 3003, 'score': 9709.232525110245, 'total_duration': 13426.772904157639, 'accumulated_submission_time': 9709.232525110245, 'accumulated_eval_time': 3680.249090194702, 'accumulated_logging_time': 0.583716869354248} +I0913 13:27:55.484449 139773553383168 logging_writer.py:48] [62578] accumulated_eval_time=3680.25, accumulated_logging_time=0.583717, accumulated_submission_time=9709.23, global_step=62578, preemption_count=0, score=9709.23, test/accuracy=0.699065, test/bleu=29.6294, test/loss=1.34706, test/num_examples=3003, total_duration=13426.8, train/accuracy=0.678117, train/bleu=34.903, train/loss=1.47351, validation/accuracy=0.682657, validation/bleu=30.0349, validation/loss=1.44898, validation/num_examples=3000 +I0913 13:29:01.138167 139773544990464 logging_writer.py:48] [63000] global_step=63000, grad_norm=0.292294, loss=1.52276 +I0913 13:29:01.141385 139802006037696 submission.py:307] 63000) loss = 1.523, grad_norm = 0.292 +I0913 13:30:18.194683 139773553383168 logging_writer.py:48] [63500] global_step=63500, grad_norm=0.314011, loss=1.60481 +I0913 13:30:18.197901 139802006037696 submission.py:307] 63500) loss = 1.605, grad_norm = 0.314 +I0913 13:31:35.269720 139773544990464 logging_writer.py:48] [64000] global_step=64000, grad_norm=0.311552, loss=1.62566 +I0913 13:31:35.273095 139802006037696 submission.py:307] 64000) loss = 1.626, grad_norm = 0.312 +I0913 13:32:52.440750 139773553383168 logging_writer.py:48] [64500] global_step=64500, grad_norm=0.331633, loss=1.62922 +I0913 13:32:52.443999 139802006037696 submission.py:307] 64500) loss = 1.629, grad_norm = 0.332 +I0913 13:34:09.603687 139773544990464 logging_writer.py:48] [65000] global_step=65000, grad_norm=0.293556, loss=1.56377 +I0913 13:34:09.606807 139802006037696 submission.py:307] 65000) loss = 1.564, grad_norm = 0.294 +I0913 13:35:26.765014 139773553383168 logging_writer.py:48] [65500] global_step=65500, grad_norm=0.344976, loss=1.58532 +I0913 13:35:26.768172 139802006037696 submission.py:307] 65500) loss = 1.585, grad_norm = 0.345 +I0913 13:36:43.946203 139773544990464 logging_writer.py:48] [66000] global_step=66000, grad_norm=0.310799, loss=1.583 +I0913 13:36:43.949342 139802006037696 submission.py:307] 66000) loss = 1.583, grad_norm = 0.311 +I0913 13:38:01.132568 139773553383168 logging_writer.py:48] [66500] global_step=66500, grad_norm=0.326137, loss=1.58156 +I0913 13:38:01.135710 139802006037696 submission.py:307] 66500) loss = 1.582, grad_norm = 0.326 +I0913 13:38:40.146520 139802006037696 spec.py:333] Evaluating on the training split. +I0913 13:38:42.347363 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 13:40:10.443429 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 13:40:12.620650 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 13:41:16.735337 139802006037696 spec.py:363] Evaluating on the test split. +I0913 13:41:18.917789 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 13:42:15.823317 139802006037696 submission_runner.py:516] Time since start: 14287.14s, Step: 66750, {'train/accuracy': 0.6760110346722221, 'train/loss': 1.4922873908265701, 'train/bleu': 33.795533798759124, 'validation/accuracy': 0.6852983843969697, 'validation/loss': 1.4414615131864452, 'validation/bleu': 30.305291905739445, 'validation/num_examples': 3000, 'test/accuracy': 0.6992969612457149, 'test/loss': 1.3391317689268492, 'test/bleu': 29.915111229994693, 'test/num_examples': 3003, 'score': 10351.489212274551, 'total_duration': 14287.140409946442, 'accumulated_submission_time': 10351.489212274551, 'accumulated_eval_time': 3895.9259502887726, 'accumulated_logging_time': 0.6218104362487793} +I0913 13:42:15.851828 139773544990464 logging_writer.py:48] [66750] accumulated_eval_time=3895.93, accumulated_logging_time=0.62181, accumulated_submission_time=10351.5, global_step=66750, preemption_count=0, score=10351.5, test/accuracy=0.699297, test/bleu=29.9151, test/loss=1.33913, test/num_examples=3003, total_duration=14287.1, train/accuracy=0.676011, train/bleu=33.7955, train/loss=1.49229, validation/accuracy=0.685298, validation/bleu=30.3053, validation/loss=1.44146, validation/num_examples=3000 +I0913 13:42:55.087190 139773553383168 logging_writer.py:48] [67000] global_step=67000, grad_norm=0.3325, loss=1.59758 +I0913 13:42:55.090481 139802006037696 submission.py:307] 67000) loss = 1.598, grad_norm = 0.332 +I0913 13:44:12.071974 139773544990464 logging_writer.py:48] [67500] global_step=67500, grad_norm=0.297389, loss=1.48556 +I0913 13:44:12.075281 139802006037696 submission.py:307] 67500) loss = 1.486, grad_norm = 0.297 +I0913 13:45:29.132994 139773553383168 logging_writer.py:48] [68000] global_step=68000, grad_norm=0.330578, loss=1.43571 +I0913 13:45:29.136106 139802006037696 submission.py:307] 68000) loss = 1.436, grad_norm = 0.331 +I0913 13:46:46.196866 139773544990464 logging_writer.py:48] [68500] global_step=68500, grad_norm=0.44665, loss=1.56043 +I0913 13:46:46.200060 139802006037696 submission.py:307] 68500) loss = 1.560, grad_norm = 0.447 +I0913 13:48:03.334660 139773553383168 logging_writer.py:48] [69000] global_step=69000, grad_norm=0.322769, loss=1.56805 +I0913 13:48:03.338030 139802006037696 submission.py:307] 69000) loss = 1.568, grad_norm = 0.323 +I0913 13:49:20.476838 139773544990464 logging_writer.py:48] [69500] global_step=69500, grad_norm=0.345504, loss=1.58364 +I0913 13:49:20.479992 139802006037696 submission.py:307] 69500) loss = 1.584, grad_norm = 0.346 +I0913 13:50:37.576394 139773553383168 logging_writer.py:48] [70000] global_step=70000, grad_norm=0.333584, loss=1.55696 +I0913 13:50:37.579471 139802006037696 submission.py:307] 70000) loss = 1.557, grad_norm = 0.334 +I0913 13:51:54.685660 139773544990464 logging_writer.py:48] [70500] global_step=70500, grad_norm=0.38266, loss=1.538 +I0913 13:51:54.688925 139802006037696 submission.py:307] 70500) loss = 1.538, grad_norm = 0.383 +I0913 13:53:00.486730 139802006037696 spec.py:333] Evaluating on the training split. +I0913 13:53:02.682839 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 13:54:24.955086 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 13:54:27.138405 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 13:55:26.856516 139802006037696 spec.py:363] Evaluating on the test split. +I0913 13:55:29.041060 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 13:56:22.585319 139802006037696 submission_runner.py:516] Time since start: 15133.90s, Step: 70924, {'train/accuracy': 0.687737344478783, 'train/loss': 1.4093970416756947, 'train/bleu': 34.64019747009979, 'validation/accuracy': 0.6861043260467942, 'validation/loss': 1.4298347393398718, 'validation/bleu': 30.260549972098406, 'validation/num_examples': 3000, 'test/accuracy': 0.7018418453314741, 'test/loss': 1.3263530590901167, 'test/bleu': 29.9305108094369, 'test/num_examples': 3003, 'score': 10993.722551584244, 'total_duration': 15133.902434110641, 'accumulated_submission_time': 10993.722551584244, 'accumulated_eval_time': 4098.024610996246, 'accumulated_logging_time': 0.6599886417388916} +I0913 13:56:22.618974 139773553383168 logging_writer.py:48] [70924] accumulated_eval_time=4098.02, accumulated_logging_time=0.659989, accumulated_submission_time=10993.7, global_step=70924, preemption_count=0, score=10993.7, test/accuracy=0.701842, test/bleu=29.9305, test/loss=1.32635, test/num_examples=3003, total_duration=15133.9, train/accuracy=0.687737, train/bleu=34.6402, train/loss=1.4094, validation/accuracy=0.686104, validation/bleu=30.2605, validation/loss=1.42983, validation/num_examples=3000 +I0913 13:56:35.077678 139773544990464 logging_writer.py:48] [71000] global_step=71000, grad_norm=0.343851, loss=1.57988 +I0913 13:56:35.080602 139802006037696 submission.py:307] 71000) loss = 1.580, grad_norm = 0.344 +I0913 13:57:51.980661 139773553383168 logging_writer.py:48] [71500] global_step=71500, grad_norm=0.354129, loss=1.5333 +I0913 13:57:51.983858 139802006037696 submission.py:307] 71500) loss = 1.533, grad_norm = 0.354 +I0913 13:59:08.859425 139773544990464 logging_writer.py:48] [72000] global_step=72000, grad_norm=0.337746, loss=1.53511 +I0913 13:59:08.862909 139802006037696 submission.py:307] 72000) loss = 1.535, grad_norm = 0.338 +I0913 14:00:25.762518 139773553383168 logging_writer.py:48] [72500] global_step=72500, grad_norm=0.346263, loss=1.53306 +I0913 14:00:25.765657 139802006037696 submission.py:307] 72500) loss = 1.533, grad_norm = 0.346 +I0913 14:01:42.633395 139773544990464 logging_writer.py:48] [73000] global_step=73000, grad_norm=0.357138, loss=1.53805 +I0913 14:01:42.636748 139802006037696 submission.py:307] 73000) loss = 1.538, grad_norm = 0.357 +I0913 14:02:59.534685 139773553383168 logging_writer.py:48] [73500] global_step=73500, grad_norm=0.5, loss=1.47308 +I0913 14:02:59.537773 139802006037696 submission.py:307] 73500) loss = 1.473, grad_norm = 0.500 +I0913 14:04:16.408287 139773544990464 logging_writer.py:48] [74000] global_step=74000, grad_norm=0.36883, loss=1.56727 +I0913 14:04:16.411537 139802006037696 submission.py:307] 74000) loss = 1.567, grad_norm = 0.369 +I0913 14:05:33.323068 139773553383168 logging_writer.py:48] [74500] global_step=74500, grad_norm=0.355393, loss=1.56266 +I0913 14:05:33.326278 139802006037696 submission.py:307] 74500) loss = 1.563, grad_norm = 0.355 +I0913 14:06:50.194212 139773544990464 logging_writer.py:48] [75000] global_step=75000, grad_norm=0.355894, loss=1.52064 +I0913 14:06:50.197449 139802006037696 submission.py:307] 75000) loss = 1.521, grad_norm = 0.356 +I0913 14:07:07.219674 139802006037696 spec.py:333] Evaluating on the training split. +I0913 14:07:09.418828 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 14:08:37.871522 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 14:08:40.052949 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 14:09:43.966187 139802006037696 spec.py:363] Evaluating on the test split. +I0913 14:09:46.145386 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 14:10:42.099635 139802006037696 submission_runner.py:516] Time since start: 15993.42s, Step: 75108, {'train/accuracy': 0.6873241369844211, 'train/loss': 1.4139003986228353, 'train/bleu': 35.10220106458447, 'validation/accuracy': 0.686401904502114, 'validation/loss': 1.4203690902778638, 'validation/bleu': 30.209283267194913, 'validation/num_examples': 3000, 'test/accuracy': 0.7033989890186508, 'test/loss': 1.3169482816222184, 'test/bleu': 30.17727112013422, 'test/num_examples': 3003, 'score': 11635.910709619522, 'total_duration': 15993.416740655899, 'accumulated_submission_time': 11635.910709619522, 'accumulated_eval_time': 4312.904577493668, 'accumulated_logging_time': 0.703080415725708} +I0913 14:10:42.128058 139773553383168 logging_writer.py:48] [75108] accumulated_eval_time=4312.9, accumulated_logging_time=0.70308, accumulated_submission_time=11635.9, global_step=75108, preemption_count=0, score=11635.9, test/accuracy=0.703399, test/bleu=30.1773, test/loss=1.31695, test/num_examples=3003, total_duration=15993.4, train/accuracy=0.687324, train/bleu=35.1022, train/loss=1.4139, validation/accuracy=0.686402, validation/bleu=30.2093, validation/loss=1.42037, validation/num_examples=3000 +I0913 14:11:43.005077 139773544990464 logging_writer.py:48] [75500] global_step=75500, grad_norm=0.387151, loss=1.49833 +I0913 14:11:43.008214 139802006037696 submission.py:307] 75500) loss = 1.498, grad_norm = 0.387 +I0913 14:12:59.826717 139773553383168 logging_writer.py:48] [76000] global_step=76000, grad_norm=0.372772, loss=1.52633 +I0913 14:12:59.829888 139802006037696 submission.py:307] 76000) loss = 1.526, grad_norm = 0.373 +I0913 14:14:16.669126 139773544990464 logging_writer.py:48] [76500] global_step=76500, grad_norm=0.33775, loss=1.51438 +I0913 14:14:16.672362 139802006037696 submission.py:307] 76500) loss = 1.514, grad_norm = 0.338 +I0913 14:15:33.630623 139773553383168 logging_writer.py:48] [77000] global_step=77000, grad_norm=0.363813, loss=1.51241 +I0913 14:15:33.634147 139802006037696 submission.py:307] 77000) loss = 1.512, grad_norm = 0.364 +I0913 14:16:50.612757 139773544990464 logging_writer.py:48] [77500] global_step=77500, grad_norm=0.322505, loss=1.55136 +I0913 14:16:50.615931 139802006037696 submission.py:307] 77500) loss = 1.551, grad_norm = 0.323 +I0913 14:18:07.593104 139773553383168 logging_writer.py:48] [78000] global_step=78000, grad_norm=0.364025, loss=1.53961 +I0913 14:18:07.596385 139802006037696 submission.py:307] 78000) loss = 1.540, grad_norm = 0.364 +I0913 14:19:24.563912 139773544990464 logging_writer.py:48] [78500] global_step=78500, grad_norm=0.355218, loss=1.45955 +I0913 14:19:24.566964 139802006037696 submission.py:307] 78500) loss = 1.460, grad_norm = 0.355 +I0913 14:20:41.539930 139773553383168 logging_writer.py:48] [79000] global_step=79000, grad_norm=0.35137, loss=1.52802 +I0913 14:20:41.543396 139802006037696 submission.py:307] 79000) loss = 1.528, grad_norm = 0.351 +I0913 14:21:26.743466 139802006037696 spec.py:333] Evaluating on the training split. +I0913 14:21:28.940707 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 14:22:57.771140 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 14:22:59.952905 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 14:24:12.430327 139802006037696 spec.py:363] Evaluating on the test split. +I0913 14:24:14.612231 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 14:25:25.491047 139802006037696 submission_runner.py:516] Time since start: 16876.81s, Step: 79291, {'train/accuracy': 0.6846116111662502, 'train/loss': 1.4252815662183282, 'train/bleu': 34.67072321627014, 'validation/accuracy': 0.6875674201187834, 'validation/loss': 1.4130921733766475, 'validation/bleu': 30.36090824007107, 'validation/num_examples': 3000, 'test/accuracy': 0.70403811515891, 'test/loss': 1.3051324371041775, 'test/bleu': 30.196260633743304, 'test/num_examples': 3003, 'score': 12278.122849941254, 'total_duration': 16876.808160305023, 'accumulated_submission_time': 12278.122849941254, 'accumulated_eval_time': 4551.652169466019, 'accumulated_logging_time': 0.7410016059875488} +I0913 14:25:25.522485 139773544990464 logging_writer.py:48] [79291] accumulated_eval_time=4551.65, accumulated_logging_time=0.741002, accumulated_submission_time=12278.1, global_step=79291, preemption_count=0, score=12278.1, test/accuracy=0.704038, test/bleu=30.1963, test/loss=1.30513, test/num_examples=3003, total_duration=16876.8, train/accuracy=0.684612, train/bleu=34.6707, train/loss=1.42528, validation/accuracy=0.687567, validation/bleu=30.3609, validation/loss=1.41309, validation/num_examples=3000 +I0913 14:25:58.330940 139773553383168 logging_writer.py:48] [79500] global_step=79500, grad_norm=0.355047, loss=1.55027 +I0913 14:25:58.334308 139802006037696 submission.py:307] 79500) loss = 1.550, grad_norm = 0.355 +I0913 14:27:15.079345 139773544990464 logging_writer.py:48] [80000] global_step=80000, grad_norm=0.360164, loss=1.49691 +I0913 14:27:15.082547 139802006037696 submission.py:307] 80000) loss = 1.497, grad_norm = 0.360 +I0913 14:28:31.975064 139773553383168 logging_writer.py:48] [80500] global_step=80500, grad_norm=0.368662, loss=1.55064 +I0913 14:28:31.978139 139802006037696 submission.py:307] 80500) loss = 1.551, grad_norm = 0.369 +I0913 14:29:48.866931 139773544990464 logging_writer.py:48] [81000] global_step=81000, grad_norm=0.341465, loss=1.52035 +I0913 14:29:48.872586 139802006037696 submission.py:307] 81000) loss = 1.520, grad_norm = 0.341 +I0913 14:31:05.760653 139773553383168 logging_writer.py:48] [81500] global_step=81500, grad_norm=0.347929, loss=1.51061 +I0913 14:31:05.763829 139802006037696 submission.py:307] 81500) loss = 1.511, grad_norm = 0.348 +I0913 14:32:22.685649 139773544990464 logging_writer.py:48] [82000] global_step=82000, grad_norm=0.354001, loss=1.49102 +I0913 14:32:22.688959 139802006037696 submission.py:307] 82000) loss = 1.491, grad_norm = 0.354 +I0913 14:33:39.563863 139773553383168 logging_writer.py:48] [82500] global_step=82500, grad_norm=0.360721, loss=1.53615 +I0913 14:33:39.567087 139802006037696 submission.py:307] 82500) loss = 1.536, grad_norm = 0.361 +I0913 14:34:56.480602 139773544990464 logging_writer.py:48] [83000] global_step=83000, grad_norm=0.337975, loss=1.44803 +I0913 14:34:56.483846 139802006037696 submission.py:307] 83000) loss = 1.448, grad_norm = 0.338 +I0913 14:36:10.097507 139802006037696 spec.py:333] Evaluating on the training split. +I0913 14:36:12.296958 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 14:37:43.763121 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 14:37:45.946552 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 14:38:53.652409 139802006037696 spec.py:363] Evaluating on the test split. +I0913 14:38:55.836514 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 14:39:53.595051 139802006037696 submission_runner.py:516] Time since start: 17744.91s, Step: 83476, {'train/accuracy': 0.6931581866075142, 'train/loss': 1.3781471677295625, 'train/bleu': 35.55717010011654, 'validation/accuracy': 0.6898364558405972, 'validation/loss': 1.4083055386789995, 'validation/bleu': 30.42063895411213, 'validation/num_examples': 3000, 'test/accuracy': 0.7054093312416477, 'test/loss': 1.298196102783104, 'test/bleu': 30.197715887844026, 'test/num_examples': 3003, 'score': 12920.284372329712, 'total_duration': 17744.912157297134, 'accumulated_submission_time': 12920.284372329712, 'accumulated_eval_time': 4775.149796485901, 'accumulated_logging_time': 0.7820522785186768} +I0913 14:39:53.625923 139773553383168 logging_writer.py:48] [83476] accumulated_eval_time=4775.15, accumulated_logging_time=0.782052, accumulated_submission_time=12920.3, global_step=83476, preemption_count=0, score=12920.3, test/accuracy=0.705409, test/bleu=30.1977, test/loss=1.2982, test/num_examples=3003, total_duration=17744.9, train/accuracy=0.693158, train/bleu=35.5572, train/loss=1.37815, validation/accuracy=0.689836, validation/bleu=30.4206, validation/loss=1.40831, validation/num_examples=3000 +I0913 14:39:58.099655 139773544990464 logging_writer.py:48] [83500] global_step=83500, grad_norm=0.349038, loss=1.51609 +I0913 14:39:58.102813 139802006037696 submission.py:307] 83500) loss = 1.516, grad_norm = 0.349 +I0913 14:41:14.844504 139773553383168 logging_writer.py:48] [84000] global_step=84000, grad_norm=0.346339, loss=1.50469 +I0913 14:41:14.847853 139802006037696 submission.py:307] 84000) loss = 1.505, grad_norm = 0.346 +I0913 14:42:31.690457 139773544990464 logging_writer.py:48] [84500] global_step=84500, grad_norm=0.39411, loss=1.56254 +I0913 14:42:31.693523 139802006037696 submission.py:307] 84500) loss = 1.563, grad_norm = 0.394 +I0913 14:43:48.570651 139773553383168 logging_writer.py:48] [85000] global_step=85000, grad_norm=0.335539, loss=1.42014 +I0913 14:43:48.573884 139802006037696 submission.py:307] 85000) loss = 1.420, grad_norm = 0.336 +I0913 14:45:05.436126 139773544990464 logging_writer.py:48] [85500] global_step=85500, grad_norm=0.36538, loss=1.49098 +I0913 14:45:05.439295 139802006037696 submission.py:307] 85500) loss = 1.491, grad_norm = 0.365 +I0913 14:46:22.318910 139773553383168 logging_writer.py:48] [86000] global_step=86000, grad_norm=0.357221, loss=1.51785 +I0913 14:46:22.322267 139802006037696 submission.py:307] 86000) loss = 1.518, grad_norm = 0.357 +I0913 14:47:39.230518 139773544990464 logging_writer.py:48] [86500] global_step=86500, grad_norm=0.365495, loss=1.45143 +I0913 14:47:39.233549 139802006037696 submission.py:307] 86500) loss = 1.451, grad_norm = 0.365 +I0913 14:48:56.162868 139773553383168 logging_writer.py:48] [87000] global_step=87000, grad_norm=0.331494, loss=1.40901 +I0913 14:48:56.166200 139802006037696 submission.py:307] 87000) loss = 1.409, grad_norm = 0.331 +I0913 14:50:13.070354 139773544990464 logging_writer.py:48] [87500] global_step=87500, grad_norm=0.359636, loss=1.39439 +I0913 14:50:13.073452 139802006037696 submission.py:307] 87500) loss = 1.394, grad_norm = 0.360 +I0913 14:50:38.247296 139802006037696 spec.py:333] Evaluating on the training split. +I0913 14:50:40.441010 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 14:52:19.818534 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 14:52:22.003825 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 14:53:28.462800 139802006037696 spec.py:363] Evaluating on the test split. +I0913 14:53:30.646882 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 14:54:29.830505 139802006037696 submission_runner.py:516] Time since start: 18621.15s, Step: 87661, {'train/accuracy': 0.6964027301420904, 'train/loss': 1.3526700890771528, 'train/bleu': 35.634105035055384, 'validation/accuracy': 0.6899728459659521, 'validation/loss': 1.4038192916082877, 'validation/bleu': 30.732287328799785, 'validation/num_examples': 3000, 'test/accuracy': 0.7070361977804892, 'test/loss': 1.2947403731625124, 'test/bleu': 30.371250564178208, 'test/num_examples': 3003, 'score': 13562.501360416412, 'total_duration': 18621.147614240646, 'accumulated_submission_time': 13562.501360416412, 'accumulated_eval_time': 5006.733048439026, 'accumulated_logging_time': 0.8224046230316162} +I0913 14:54:29.860316 139773553383168 logging_writer.py:48] [87661] accumulated_eval_time=5006.73, accumulated_logging_time=0.822405, accumulated_submission_time=13562.5, global_step=87661, preemption_count=0, score=13562.5, test/accuracy=0.707036, test/bleu=30.3713, test/loss=1.29474, test/num_examples=3003, total_duration=18621.1, train/accuracy=0.696403, train/bleu=35.6341, train/loss=1.35267, validation/accuracy=0.689973, validation/bleu=30.7323, validation/loss=1.40382, validation/num_examples=3000 +I0913 14:55:22.614893 139773544990464 logging_writer.py:48] [88000] global_step=88000, grad_norm=0.33186, loss=1.50903 +I0913 14:55:22.617905 139802006037696 submission.py:307] 88000) loss = 1.509, grad_norm = 0.332 +I0913 14:56:39.392411 139773553383168 logging_writer.py:48] [88500] global_step=88500, grad_norm=0.364734, loss=1.47807 +I0913 14:56:39.395836 139802006037696 submission.py:307] 88500) loss = 1.478, grad_norm = 0.365 +I0913 14:57:56.255072 139773544990464 logging_writer.py:48] [89000] global_step=89000, grad_norm=0.339362, loss=1.38229 +I0913 14:57:56.258171 139802006037696 submission.py:307] 89000) loss = 1.382, grad_norm = 0.339 +I0913 14:59:13.098115 139773553383168 logging_writer.py:48] [89500] global_step=89500, grad_norm=0.32006, loss=1.49119 +I0913 14:59:13.101271 139802006037696 submission.py:307] 89500) loss = 1.491, grad_norm = 0.320 +I0913 15:00:29.963102 139773544990464 logging_writer.py:48] [90000] global_step=90000, grad_norm=0.343884, loss=1.53582 +I0913 15:00:29.966269 139802006037696 submission.py:307] 90000) loss = 1.536, grad_norm = 0.344 +I0913 15:01:46.832892 139773553383168 logging_writer.py:48] [90500] global_step=90500, grad_norm=0.350839, loss=1.51913 +I0913 15:01:46.836144 139802006037696 submission.py:307] 90500) loss = 1.519, grad_norm = 0.351 +I0913 15:03:03.780398 139773544990464 logging_writer.py:48] [91000] global_step=91000, grad_norm=0.368232, loss=1.49687 +I0913 15:03:03.783423 139802006037696 submission.py:307] 91000) loss = 1.497, grad_norm = 0.368 +I0913 15:04:20.713047 139773553383168 logging_writer.py:48] [91500] global_step=91500, grad_norm=0.348267, loss=1.52341 +I0913 15:04:20.716388 139802006037696 submission.py:307] 91500) loss = 1.523, grad_norm = 0.348 +I0913 15:05:14.483206 139802006037696 spec.py:333] Evaluating on the training split. +I0913 15:05:16.680207 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 15:06:51.648522 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 15:06:53.830966 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 15:07:56.328042 139802006037696 spec.py:363] Evaluating on the test split. +I0913 15:07:58.504523 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 15:08:56.121170 139802006037696 submission_runner.py:516] Time since start: 19487.44s, Step: 91847, {'train/accuracy': 0.6967524734334921, 'train/loss': 1.3594542629511726, 'train/bleu': 35.62226473493535, 'validation/accuracy': 0.6922542807900708, 'validation/loss': 1.3970977188751534, 'validation/bleu': 30.529455857116726, 'validation/num_examples': 3000, 'test/accuracy': 0.7080820405554588, 'test/loss': 1.2840292908604962, 'test/bleu': 30.630104490928616, 'test/num_examples': 3003, 'score': 14204.70498752594, 'total_duration': 19487.438278913498, 'accumulated_submission_time': 14204.70498752594, 'accumulated_eval_time': 5228.371054410934, 'accumulated_logging_time': 0.8618402481079102} +I0913 15:08:56.151101 139773544990464 logging_writer.py:48] [91847] accumulated_eval_time=5228.37, accumulated_logging_time=0.86184, accumulated_submission_time=14204.7, global_step=91847, preemption_count=0, score=14204.7, test/accuracy=0.708082, test/bleu=30.6301, test/loss=1.28403, test/num_examples=3003, total_duration=19487.4, train/accuracy=0.696752, train/bleu=35.6223, train/loss=1.35945, validation/accuracy=0.692254, validation/bleu=30.5295, validation/loss=1.3971, validation/num_examples=3000 +I0913 15:09:20.400914 139773553383168 logging_writer.py:48] [92000] global_step=92000, grad_norm=0.369485, loss=1.51206 +I0913 15:09:20.403950 139802006037696 submission.py:307] 92000) loss = 1.512, grad_norm = 0.369 +I0913 15:10:37.193659 139773544990464 logging_writer.py:48] [92500] global_step=92500, grad_norm=0.327804, loss=1.42388 +I0913 15:10:37.196866 139802006037696 submission.py:307] 92500) loss = 1.424, grad_norm = 0.328 +I0913 15:11:54.055802 139773553383168 logging_writer.py:48] [93000] global_step=93000, grad_norm=0.370987, loss=1.47366 +I0913 15:11:54.059142 139802006037696 submission.py:307] 93000) loss = 1.474, grad_norm = 0.371 +I0913 15:13:10.922302 139773544990464 logging_writer.py:48] [93500] global_step=93500, grad_norm=0.347172, loss=1.45474 +I0913 15:13:10.925388 139802006037696 submission.py:307] 93500) loss = 1.455, grad_norm = 0.347 +I0913 15:14:27.825270 139773553383168 logging_writer.py:48] [94000] global_step=94000, grad_norm=0.383163, loss=1.53647 +I0913 15:14:27.828434 139802006037696 submission.py:307] 94000) loss = 1.536, grad_norm = 0.383 +I0913 15:15:44.689320 139773544990464 logging_writer.py:48] [94500] global_step=94500, grad_norm=0.356017, loss=1.47775 +I0913 15:15:44.692483 139802006037696 submission.py:307] 94500) loss = 1.478, grad_norm = 0.356 +I0913 15:17:01.589755 139773553383168 logging_writer.py:48] [95000] global_step=95000, grad_norm=0.335522, loss=1.48088 +I0913 15:17:01.593093 139802006037696 submission.py:307] 95000) loss = 1.481, grad_norm = 0.336 +I0913 15:18:18.524914 139773544990464 logging_writer.py:48] [95500] global_step=95500, grad_norm=0.32689, loss=1.46042 +I0913 15:18:18.528337 139802006037696 submission.py:307] 95500) loss = 1.460, grad_norm = 0.327 +I0913 15:19:35.438533 139773553383168 logging_writer.py:48] [96000] global_step=96000, grad_norm=0.343582, loss=1.42479 +I0913 15:19:35.441680 139802006037696 submission.py:307] 96000) loss = 1.425, grad_norm = 0.344 +I0913 15:19:40.783651 139802006037696 spec.py:333] Evaluating on the training split. +I0913 15:19:42.978609 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 15:21:08.864370 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 15:21:11.048120 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 15:22:11.258853 139802006037696 spec.py:363] Evaluating on the test split. +I0913 15:22:13.437199 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 15:23:07.391369 139802006037696 submission_runner.py:516] Time since start: 20338.71s, Step: 96032, {'train/accuracy': 0.699206924732668, 'train/loss': 1.3320539563117972, 'train/bleu': 36.02363252187802, 'validation/accuracy': 0.6921674870739358, 'validation/loss': 1.3897272274987291, 'validation/bleu': 30.503563924140668, 'validation/num_examples': 3000, 'test/accuracy': 0.7093486723606995, 'test/loss': 1.2777466990587416, 'test/bleu': 30.37390140044731, 'test/num_examples': 3003, 'score': 14846.929310798645, 'total_duration': 20338.708476305008, 'accumulated_submission_time': 14846.929310798645, 'accumulated_eval_time': 5434.9788410663605, 'accumulated_logging_time': 0.9011752605438232} +I0913 15:23:07.421554 139773544990464 logging_writer.py:48] [96032] accumulated_eval_time=5434.98, accumulated_logging_time=0.901175, accumulated_submission_time=14846.9, global_step=96032, preemption_count=0, score=14846.9, test/accuracy=0.709349, test/bleu=30.3739, test/loss=1.27775, test/num_examples=3003, total_duration=20338.7, train/accuracy=0.699207, train/bleu=36.0236, train/loss=1.33205, validation/accuracy=0.692167, validation/bleu=30.5036, validation/loss=1.38973, validation/num_examples=3000 +I0913 15:24:19.959322 139773553383168 logging_writer.py:48] [96500] global_step=96500, grad_norm=0.339783, loss=1.46473 +I0913 15:24:19.962687 139802006037696 submission.py:307] 96500) loss = 1.465, grad_norm = 0.340 +I0913 15:25:36.772670 139773544990464 logging_writer.py:48] [97000] global_step=97000, grad_norm=0.347553, loss=1.51136 +I0913 15:25:36.775804 139802006037696 submission.py:307] 97000) loss = 1.511, grad_norm = 0.348 +I0913 15:26:53.634217 139773553383168 logging_writer.py:48] [97500] global_step=97500, grad_norm=0.328583, loss=1.46113 +I0913 15:26:53.637359 139802006037696 submission.py:307] 97500) loss = 1.461, grad_norm = 0.329 +I0913 15:28:10.555289 139773544990464 logging_writer.py:48] [98000] global_step=98000, grad_norm=0.336552, loss=1.46213 +I0913 15:28:10.558846 139802006037696 submission.py:307] 98000) loss = 1.462, grad_norm = 0.337 +I0913 15:29:27.437585 139773553383168 logging_writer.py:48] [98500] global_step=98500, grad_norm=0.344952, loss=1.37032 +I0913 15:29:27.440912 139802006037696 submission.py:307] 98500) loss = 1.370, grad_norm = 0.345 +I0913 15:30:44.321089 139773544990464 logging_writer.py:48] [99000] global_step=99000, grad_norm=0.317176, loss=1.38889 +I0913 15:30:44.324153 139802006037696 submission.py:307] 99000) loss = 1.389, grad_norm = 0.317 +I0913 15:32:01.190916 139773553383168 logging_writer.py:48] [99500] global_step=99500, grad_norm=0.351444, loss=1.38449 +I0913 15:32:01.194252 139802006037696 submission.py:307] 99500) loss = 1.384, grad_norm = 0.351 +I0913 15:33:18.084681 139773544990464 logging_writer.py:48] [100000] global_step=100000, grad_norm=0.372665, loss=1.41194 +I0913 15:33:18.088006 139802006037696 submission.py:307] 100000) loss = 1.412, grad_norm = 0.373 +I0913 15:33:52.025094 139802006037696 spec.py:333] Evaluating on the training split. +I0913 15:33:54.222754 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 15:35:24.006523 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 15:35:26.187762 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 15:36:30.273487 139802006037696 spec.py:363] Evaluating on the test split. +I0913 15:36:32.452405 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 15:37:27.159161 139802006037696 submission_runner.py:516] Time since start: 21198.48s, Step: 100218, {'train/accuracy': 0.7010576275827448, 'train/loss': 1.32520942908647, 'train/bleu': 36.29287865890685, 'validation/accuracy': 0.6933825990998251, 'validation/loss': 1.387492773648188, 'validation/bleu': 30.703973921269075, 'validation/num_examples': 3000, 'test/accuracy': 0.7110568822264831, 'test/loss': 1.2748164328046017, 'test/bleu': 30.83824125061178, 'test/num_examples': 3003, 'score': 15489.126465082169, 'total_duration': 21198.476254463196, 'accumulated_submission_time': 15489.126465082169, 'accumulated_eval_time': 5650.112952947617, 'accumulated_logging_time': 0.9410097599029541} +I0913 15:37:27.190585 139773553383168 logging_writer.py:48] [100218] accumulated_eval_time=5650.11, accumulated_logging_time=0.94101, accumulated_submission_time=15489.1, global_step=100218, preemption_count=0, score=15489.1, test/accuracy=0.711057, test/bleu=30.8382, test/loss=1.27482, test/num_examples=3003, total_duration=21198.5, train/accuracy=0.701058, train/bleu=36.2929, train/loss=1.32521, validation/accuracy=0.693383, validation/bleu=30.704, validation/loss=1.38749, validation/num_examples=3000 +I0913 15:38:11.235254 139773544990464 logging_writer.py:48] [100500] global_step=100500, grad_norm=0.372328, loss=1.4839 +I0913 15:38:11.238334 139802006037696 submission.py:307] 100500) loss = 1.484, grad_norm = 0.372 +I0913 15:39:28.020998 139773553383168 logging_writer.py:48] [101000] global_step=101000, grad_norm=0.366359, loss=1.41132 +I0913 15:39:28.024280 139802006037696 submission.py:307] 101000) loss = 1.411, grad_norm = 0.366 +I0913 15:40:44.856672 139773544990464 logging_writer.py:48] [101500] global_step=101500, grad_norm=0.343346, loss=1.44566 +I0913 15:40:44.859944 139802006037696 submission.py:307] 101500) loss = 1.446, grad_norm = 0.343 +I0913 15:42:01.714610 139773553383168 logging_writer.py:48] [102000] global_step=102000, grad_norm=0.339426, loss=1.4298 +I0913 15:42:01.717710 139802006037696 submission.py:307] 102000) loss = 1.430, grad_norm = 0.339 +I0913 15:43:18.570784 139773544990464 logging_writer.py:48] [102500] global_step=102500, grad_norm=0.336475, loss=1.34102 +I0913 15:43:18.573950 139802006037696 submission.py:307] 102500) loss = 1.341, grad_norm = 0.336 +I0913 15:44:35.456885 139773553383168 logging_writer.py:48] [103000] global_step=103000, grad_norm=0.349869, loss=1.40059 +I0913 15:44:35.460261 139802006037696 submission.py:307] 103000) loss = 1.401, grad_norm = 0.350 +I0913 15:45:52.319458 139773544990464 logging_writer.py:48] [103500] global_step=103500, grad_norm=0.333105, loss=1.37184 +I0913 15:45:52.322708 139802006037696 submission.py:307] 103500) loss = 1.372, grad_norm = 0.333 +I0913 15:47:09.220553 139773553383168 logging_writer.py:48] [104000] global_step=104000, grad_norm=0.365076, loss=1.49164 +I0913 15:47:09.223578 139802006037696 submission.py:307] 104000) loss = 1.492, grad_norm = 0.365 +I0913 15:48:11.769829 139802006037696 spec.py:333] Evaluating on the training split. +I0913 15:48:13.966827 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 15:49:48.502207 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 15:49:50.678698 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 15:50:55.902341 139802006037696 spec.py:363] Evaluating on the test split. +I0913 15:50:58.082163 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 15:52:00.145800 139802006037696 submission_runner.py:516] Time since start: 22071.46s, Step: 104404, {'train/accuracy': 0.7013492118124871, 'train/loss': 1.3243644019182672, 'train/bleu': 36.5867640368117, 'validation/accuracy': 0.693580984736705, 'validation/loss': 1.3883863808260282, 'validation/bleu': 30.800434632280638, 'validation/num_examples': 3000, 'test/accuracy': 0.7112544303061995, 'test/loss': 1.2717339201673348, 'test/bleu': 30.82440532248409, 'test/num_examples': 3003, 'score': 16131.295519828796, 'total_duration': 22071.462913513184, 'accumulated_submission_time': 16131.295519828796, 'accumulated_eval_time': 5878.489018201828, 'accumulated_logging_time': 0.9821929931640625} +I0913 15:52:00.176135 139773544990464 logging_writer.py:48] [104404] accumulated_eval_time=5878.49, accumulated_logging_time=0.982193, accumulated_submission_time=16131.3, global_step=104404, preemption_count=0, score=16131.3, test/accuracy=0.711254, test/bleu=30.8244, test/loss=1.27173, test/num_examples=3003, total_duration=22071.5, train/accuracy=0.701349, train/bleu=36.5868, train/loss=1.32436, validation/accuracy=0.693581, validation/bleu=30.8004, validation/loss=1.38839, validation/num_examples=3000 +I0913 15:52:15.664737 139773553383168 logging_writer.py:48] [104500] global_step=104500, grad_norm=0.341801, loss=1.42704 +I0913 15:52:15.667646 139802006037696 submission.py:307] 104500) loss = 1.427, grad_norm = 0.342 +I0913 15:53:32.379872 139773544990464 logging_writer.py:48] [105000] global_step=105000, grad_norm=0.360831, loss=1.43925 +I0913 15:53:32.382963 139802006037696 submission.py:307] 105000) loss = 1.439, grad_norm = 0.361 +I0913 15:54:49.234624 139773553383168 logging_writer.py:48] [105500] global_step=105500, grad_norm=0.333867, loss=1.43653 +I0913 15:54:49.238035 139802006037696 submission.py:307] 105500) loss = 1.437, grad_norm = 0.334 +I0913 15:56:06.100602 139773544990464 logging_writer.py:48] [106000] global_step=106000, grad_norm=0.342085, loss=1.47139 +I0913 15:56:06.103931 139802006037696 submission.py:307] 106000) loss = 1.471, grad_norm = 0.342 +I0913 15:57:23.003909 139773553383168 logging_writer.py:48] [106500] global_step=106500, grad_norm=0.326775, loss=1.38367 +I0913 15:57:23.006949 139802006037696 submission.py:307] 106500) loss = 1.384, grad_norm = 0.327 +I0913 15:58:40.035144 139773544990464 logging_writer.py:48] [107000] global_step=107000, grad_norm=0.331032, loss=1.45626 +I0913 15:58:40.038286 139802006037696 submission.py:307] 107000) loss = 1.456, grad_norm = 0.331 +I0913 15:59:57.017838 139773553383168 logging_writer.py:48] [107500] global_step=107500, grad_norm=0.348706, loss=1.43694 +I0913 15:59:57.021153 139802006037696 submission.py:307] 107500) loss = 1.437, grad_norm = 0.349 +I0913 16:01:13.999851 139773544990464 logging_writer.py:48] [108000] global_step=108000, grad_norm=0.338576, loss=1.43661 +I0913 16:01:14.003006 139802006037696 submission.py:307] 108000) loss = 1.437, grad_norm = 0.339 +I0913 16:02:30.984609 139773553383168 logging_writer.py:48] [108500] global_step=108500, grad_norm=0.352023, loss=1.36021 +I0913 16:02:30.987707 139802006037696 submission.py:307] 108500) loss = 1.360, grad_norm = 0.352 +I0913 16:02:44.794786 139802006037696 spec.py:333] Evaluating on the training split. +I0913 16:02:46.992308 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 16:04:16.959806 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 16:04:19.136011 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 16:05:24.247833 139802006037696 spec.py:363] Evaluating on the test split. +I0913 16:05:26.429370 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 16:06:27.694663 139802006037696 submission_runner.py:516] Time since start: 22939.01s, Step: 108587, {'train/accuracy': 0.7014952301481698, 'train/loss': 1.3235886016891816, 'train/bleu': 36.552775460332526, 'validation/accuracy': 0.6941885407496498, 'validation/loss': 1.3850052037482485, 'validation/bleu': 30.741540601810694, 'validation/num_examples': 3000, 'test/accuracy': 0.710789611295102, 'test/loss': 1.2702854460809947, 'test/bleu': 30.62603422876235, 'test/num_examples': 3003, 'score': 16773.50749373436, 'total_duration': 22939.01176381111, 'accumulated_submission_time': 16773.50749373436, 'accumulated_eval_time': 6101.3889355659485, 'accumulated_logging_time': 1.0220787525177002} +I0913 16:06:27.727411 139773544990464 logging_writer.py:48] [108587] accumulated_eval_time=6101.39, accumulated_logging_time=1.02208, accumulated_submission_time=16773.5, global_step=108587, preemption_count=0, score=16773.5, test/accuracy=0.71079, test/bleu=30.626, test/loss=1.27029, test/num_examples=3003, total_duration=22939, train/accuracy=0.701495, train/bleu=36.5528, train/loss=1.32359, validation/accuracy=0.694189, validation/bleu=30.7415, validation/loss=1.38501, validation/num_examples=3000 +I0913 16:07:31.862254 139773553383168 logging_writer.py:48] [109000] global_step=109000, grad_norm=0.367022, loss=1.46826 +I0913 16:07:31.865379 139802006037696 submission.py:307] 109000) loss = 1.468, grad_norm = 0.367 +I0913 16:08:48.731692 139773544990464 logging_writer.py:48] [109500] global_step=109500, grad_norm=0.34298, loss=1.3466 +I0913 16:08:48.735255 139802006037696 submission.py:307] 109500) loss = 1.347, grad_norm = 0.343 +I0913 16:10:05.609179 139773553383168 logging_writer.py:48] [110000] global_step=110000, grad_norm=0.329696, loss=1.39084 +I0913 16:10:05.612318 139802006037696 submission.py:307] 110000) loss = 1.391, grad_norm = 0.330 +I0913 16:11:22.490898 139773544990464 logging_writer.py:48] [110500] global_step=110500, grad_norm=0.339035, loss=1.4822 +I0913 16:11:22.494170 139802006037696 submission.py:307] 110500) loss = 1.482, grad_norm = 0.339 +I0913 16:12:39.358368 139773553383168 logging_writer.py:48] [111000] global_step=111000, grad_norm=0.353224, loss=1.40453 +I0913 16:12:39.361539 139802006037696 submission.py:307] 111000) loss = 1.405, grad_norm = 0.353 +I0913 16:13:56.241118 139773544990464 logging_writer.py:48] [111500] global_step=111500, grad_norm=0.349326, loss=1.36533 +I0913 16:13:56.244259 139802006037696 submission.py:307] 111500) loss = 1.365, grad_norm = 0.349 +I0913 16:15:13.151975 139773553383168 logging_writer.py:48] [112000] global_step=112000, grad_norm=0.325753, loss=1.39353 +I0913 16:15:13.155086 139802006037696 submission.py:307] 112000) loss = 1.394, grad_norm = 0.326 +I0913 16:16:30.037614 139773544990464 logging_writer.py:48] [112500] global_step=112500, grad_norm=0.319961, loss=1.30624 +I0913 16:16:30.040915 139802006037696 submission.py:307] 112500) loss = 1.306, grad_norm = 0.320 +I0913 16:17:12.410130 139802006037696 spec.py:333] Evaluating on the training split. +I0913 16:17:14.608983 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 16:18:42.955309 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 16:18:45.132526 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 16:19:47.801752 139802006037696 spec.py:363] Evaluating on the test split. +I0913 16:19:49.976826 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 16:20:45.811009 139802006037696 submission_runner.py:516] Time since start: 23797.13s, Step: 112773, {'train/accuracy': 0.700723999496746, 'train/loss': 1.3307948732429, 'train/bleu': 36.62687539942267, 'validation/accuracy': 0.6941637425450398, 'validation/loss': 1.3828152123036292, 'validation/bleu': 30.74221672807699, 'validation/num_examples': 3000, 'test/accuracy': 0.711905176921736, 'test/loss': 1.2670732017314508, 'test/bleu': 30.748248276620576, 'test/num_examples': 3003, 'score': 17415.770446300507, 'total_duration': 23797.128108739853, 'accumulated_submission_time': 17415.770446300507, 'accumulated_eval_time': 6314.789815425873, 'accumulated_logging_time': 1.0644431114196777} +I0913 16:20:45.842215 139773553383168 logging_writer.py:48] [112773] accumulated_eval_time=6314.79, accumulated_logging_time=1.06444, accumulated_submission_time=17415.8, global_step=112773, preemption_count=0, score=17415.8, test/accuracy=0.711905, test/bleu=30.7482, test/loss=1.26707, test/num_examples=3003, total_duration=23797.1, train/accuracy=0.700724, train/bleu=36.6269, train/loss=1.33079, validation/accuracy=0.694164, validation/bleu=30.7422, validation/loss=1.38282, validation/num_examples=3000 +I0913 16:21:21.431308 139773544990464 logging_writer.py:48] [113000] global_step=113000, grad_norm=0.349983, loss=1.39181 +I0913 16:21:21.434374 139802006037696 submission.py:307] 113000) loss = 1.392, grad_norm = 0.350 +I0913 16:22:38.217868 139773553383168 logging_writer.py:48] [113500] global_step=113500, grad_norm=0.348479, loss=1.32679 +I0913 16:22:38.221105 139802006037696 submission.py:307] 113500) loss = 1.327, grad_norm = 0.348 +I0913 16:23:55.081324 139773544990464 logging_writer.py:48] [114000] global_step=114000, grad_norm=0.333718, loss=1.40515 +I0913 16:23:55.084522 139802006037696 submission.py:307] 114000) loss = 1.405, grad_norm = 0.334 +I0913 16:25:11.980685 139773553383168 logging_writer.py:48] [114500] global_step=114500, grad_norm=0.325113, loss=1.44569 +I0913 16:25:11.983838 139802006037696 submission.py:307] 114500) loss = 1.446, grad_norm = 0.325 +I0913 16:26:28.874560 139773544990464 logging_writer.py:48] [115000] global_step=115000, grad_norm=0.362622, loss=1.37365 +I0913 16:26:28.877765 139802006037696 submission.py:307] 115000) loss = 1.374, grad_norm = 0.363 +I0913 16:27:45.769628 139773553383168 logging_writer.py:48] [115500] global_step=115500, grad_norm=0.343299, loss=1.49749 +I0913 16:27:45.772862 139802006037696 submission.py:307] 115500) loss = 1.497, grad_norm = 0.343 +I0913 16:29:02.708819 139773544990464 logging_writer.py:48] [116000] global_step=116000, grad_norm=0.357114, loss=1.42985 +I0913 16:29:02.712190 139802006037696 submission.py:307] 116000) loss = 1.430, grad_norm = 0.357 +I0913 16:30:19.653458 139773553383168 logging_writer.py:48] [116500] global_step=116500, grad_norm=0.335378, loss=1.43215 +I0913 16:30:19.656606 139802006037696 submission.py:307] 116500) loss = 1.432, grad_norm = 0.335 +I0913 16:31:30.503608 139802006037696 spec.py:333] Evaluating on the training split. +I0913 16:31:32.703496 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 16:32:54.588553 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 16:32:56.769515 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 16:33:57.377212 139802006037696 spec.py:363] Evaluating on the test split. +I0913 16:33:59.551968 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 16:34:55.118017 139802006037696 submission_runner.py:516] Time since start: 24646.44s, Step: 116958, {'train/accuracy': 0.7042175616501498, 'train/loss': 1.3043353163171238, 'train/bleu': 36.602375940877536, 'validation/accuracy': 0.6942009398519547, 'validation/loss': 1.383598196240592, 'validation/bleu': 30.718576609299046, 'validation/num_examples': 3000, 'test/accuracy': 0.7116030445645227, 'test/loss': 1.2678339797222706, 'test/bleu': 30.769477198529625, 'test/num_examples': 3003, 'score': 18058.02095270157, 'total_duration': 24646.43512058258, 'accumulated_submission_time': 18058.02095270157, 'accumulated_eval_time': 6519.404279708862, 'accumulated_logging_time': 1.1050832271575928} +I0913 16:34:55.148674 139773544990464 logging_writer.py:48] [116958] accumulated_eval_time=6519.4, accumulated_logging_time=1.10508, accumulated_submission_time=18058, global_step=116958, preemption_count=0, score=18058, test/accuracy=0.711603, test/bleu=30.7695, test/loss=1.26783, test/num_examples=3003, total_duration=24646.4, train/accuracy=0.704218, train/bleu=36.6024, train/loss=1.30434, validation/accuracy=0.694201, validation/bleu=30.7186, validation/loss=1.3836, validation/num_examples=3000 +I0913 16:35:02.380581 139773553383168 logging_writer.py:48] [117000] global_step=117000, grad_norm=0.332718, loss=1.40006 +I0913 16:35:02.383580 139802006037696 submission.py:307] 117000) loss = 1.400, grad_norm = 0.333 +I0913 16:36:19.114943 139773544990464 logging_writer.py:48] [117500] global_step=117500, grad_norm=0.340633, loss=1.47888 +I0913 16:36:19.118183 139802006037696 submission.py:307] 117500) loss = 1.479, grad_norm = 0.341 +I0913 16:37:36.012862 139773553383168 logging_writer.py:48] [118000] global_step=118000, grad_norm=0.352405, loss=1.41867 +I0913 16:37:36.016000 139802006037696 submission.py:307] 118000) loss = 1.419, grad_norm = 0.352 +I0913 16:38:52.949151 139773544990464 logging_writer.py:48] [118500] global_step=118500, grad_norm=0.339328, loss=1.35808 +I0913 16:38:52.952560 139802006037696 submission.py:307] 118500) loss = 1.358, grad_norm = 0.339 +I0913 16:40:09.855385 139773553383168 logging_writer.py:48] [119000] global_step=119000, grad_norm=0.336774, loss=1.41793 +I0913 16:40:09.858554 139802006037696 submission.py:307] 119000) loss = 1.418, grad_norm = 0.337 +I0913 16:41:26.810835 139773544990464 logging_writer.py:48] [119500] global_step=119500, grad_norm=0.32491, loss=1.39803 +I0913 16:41:26.813992 139802006037696 submission.py:307] 119500) loss = 1.398, grad_norm = 0.325 +I0913 16:42:43.769178 139773553383168 logging_writer.py:48] [120000] global_step=120000, grad_norm=0.33126, loss=1.32541 +I0913 16:42:43.772282 139802006037696 submission.py:307] 120000) loss = 1.325, grad_norm = 0.331 +I0913 16:44:00.733532 139773544990464 logging_writer.py:48] [120500] global_step=120500, grad_norm=0.344154, loss=1.36306 +I0913 16:44:00.736908 139802006037696 submission.py:307] 120500) loss = 1.363, grad_norm = 0.344 +I0913 16:45:17.682498 139773553383168 logging_writer.py:48] [121000] global_step=121000, grad_norm=0.327863, loss=1.44237 +I0913 16:45:17.685590 139802006037696 submission.py:307] 121000) loss = 1.442, grad_norm = 0.328 +I0913 16:45:39.783801 139802006037696 spec.py:333] Evaluating on the training split. +I0913 16:45:41.983230 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 16:47:10.321232 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 16:47:12.502031 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 16:48:12.946843 139802006037696 spec.py:363] Evaluating on the test split. +I0913 16:48:15.129683 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 16:49:09.939256 139802006037696 submission_runner.py:516] Time since start: 25501.26s, Step: 121141, {'train/accuracy': 0.7014666438395252, 'train/loss': 1.3255141456942305, 'train/bleu': 36.53351868484419, 'validation/accuracy': 0.6942257380565647, 'validation/loss': 1.3836071080953738, 'validation/bleu': 30.655805237988723, 'validation/num_examples': 3000, 'test/accuracy': 0.7117192493172971, 'test/loss': 1.267700162686654, 'test/bleu': 30.775598462168684, 'test/num_examples': 3003, 'score': 18700.251742362976, 'total_duration': 25501.25637102127, 'accumulated_submission_time': 18700.251742362976, 'accumulated_eval_time': 6729.55979180336, 'accumulated_logging_time': 1.1453070640563965} +I0913 16:49:09.972160 139773544990464 logging_writer.py:48] [121141] accumulated_eval_time=6729.56, accumulated_logging_time=1.14531, accumulated_submission_time=18700.3, global_step=121141, preemption_count=0, score=18700.3, test/accuracy=0.711719, test/bleu=30.7756, test/loss=1.2677, test/num_examples=3003, total_duration=25501.3, train/accuracy=0.701467, train/bleu=36.5335, train/loss=1.32551, validation/accuracy=0.694226, validation/bleu=30.6558, validation/loss=1.38361, validation/num_examples=3000 +I0913 16:50:05.873381 139773553383168 logging_writer.py:48] [121500] global_step=121500, grad_norm=0.337266, loss=1.45949 +I0913 16:50:05.876648 139802006037696 submission.py:307] 121500) loss = 1.459, grad_norm = 0.337 +I0913 16:51:22.777983 139773544990464 logging_writer.py:48] [122000] global_step=122000, grad_norm=0.346373, loss=1.44663 +I0913 16:51:22.781160 139802006037696 submission.py:307] 122000) loss = 1.447, grad_norm = 0.346 +I0913 16:52:39.749538 139773553383168 logging_writer.py:48] [122500] global_step=122500, grad_norm=0.347512, loss=1.35876 +I0913 16:52:39.752717 139802006037696 submission.py:307] 122500) loss = 1.359, grad_norm = 0.348 +I0913 16:53:56.734202 139773544990464 logging_writer.py:48] [123000] global_step=123000, grad_norm=0.330483, loss=1.43373 +I0913 16:53:56.737348 139802006037696 submission.py:307] 123000) loss = 1.434, grad_norm = 0.330 +I0913 16:55:13.736859 139773553383168 logging_writer.py:48] [123500] global_step=123500, grad_norm=0.338289, loss=1.35124 +I0913 16:55:13.740167 139802006037696 submission.py:307] 123500) loss = 1.351, grad_norm = 0.338 +I0913 16:56:30.762255 139773544990464 logging_writer.py:48] [124000] global_step=124000, grad_norm=0.346241, loss=1.46529 +I0913 16:56:30.765569 139802006037696 submission.py:307] 124000) loss = 1.465, grad_norm = 0.346 +I0913 16:57:47.792515 139773553383168 logging_writer.py:48] [124500] global_step=124500, grad_norm=0.344262, loss=1.34559 +I0913 16:57:47.795723 139802006037696 submission.py:307] 124500) loss = 1.346, grad_norm = 0.344 +I0913 16:59:04.833488 139773544990464 logging_writer.py:48] [125000] global_step=125000, grad_norm=0.32764, loss=1.36624 +I0913 16:59:04.836754 139802006037696 submission.py:307] 125000) loss = 1.366, grad_norm = 0.328 +I0913 16:59:54.537892 139802006037696 spec.py:333] Evaluating on the training split. +I0913 16:59:56.736907 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 17:01:24.753466 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 17:01:26.926775 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 17:02:27.959862 139802006037696 spec.py:363] Evaluating on the test split. +I0913 17:02:30.138441 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 17:03:25.501516 139802006037696 submission_runner.py:516] Time since start: 26356.82s, Step: 125320, {'train/accuracy': 0.7062242562929062, 'train/loss': 1.2931648669908467, 'train/bleu': 36.344055673993815, 'validation/accuracy': 0.6943249308750047, 'validation/loss': 1.3830785963596235, 'validation/bleu': 30.692571533837658, 'validation/num_examples': 3000, 'test/accuracy': 0.7117657312184068, 'test/loss': 1.2675796002556505, 'test/bleu': 30.767489851609625, 'test/num_examples': 3003, 'score': 19342.42072749138, 'total_duration': 26356.818597316742, 'accumulated_submission_time': 19342.42072749138, 'accumulated_eval_time': 6940.523397922516, 'accumulated_logging_time': 1.1880042552947998} +I0913 17:03:25.535073 139773553383168 logging_writer.py:48] [125320] accumulated_eval_time=6940.52, accumulated_logging_time=1.188, accumulated_submission_time=19342.4, global_step=125320, preemption_count=0, score=19342.4, test/accuracy=0.711766, test/bleu=30.7675, test/loss=1.26758, test/num_examples=3003, total_duration=26356.8, train/accuracy=0.706224, train/bleu=36.3441, train/loss=1.29316, validation/accuracy=0.694325, validation/bleu=30.6926, validation/loss=1.38308, validation/num_examples=3000 +I0913 17:03:53.913864 139773544990464 logging_writer.py:48] [125500] global_step=125500, grad_norm=0.332191, loss=1.40439 +I0913 17:03:53.916788 139802006037696 submission.py:307] 125500) loss = 1.404, grad_norm = 0.332 +I0913 17:05:10.651710 139773553383168 logging_writer.py:48] [126000] global_step=126000, grad_norm=0.346685, loss=1.41104 +I0913 17:05:10.655156 139802006037696 submission.py:307] 126000) loss = 1.411, grad_norm = 0.347 +I0913 17:06:27.498572 139773544990464 logging_writer.py:48] [126500] global_step=126500, grad_norm=0.371941, loss=1.51337 +I0913 17:06:27.501588 139802006037696 submission.py:307] 126500) loss = 1.513, grad_norm = 0.372 +I0913 17:07:44.399853 139773553383168 logging_writer.py:48] [127000] global_step=127000, grad_norm=0.326611, loss=1.33121 +I0913 17:07:44.403106 139802006037696 submission.py:307] 127000) loss = 1.331, grad_norm = 0.327 +I0913 17:09:01.335353 139773544990464 logging_writer.py:48] [127500] global_step=127500, grad_norm=0.357425, loss=1.3984 +I0913 17:09:01.338504 139802006037696 submission.py:307] 127500) loss = 1.398, grad_norm = 0.357 +I0913 17:10:18.243422 139773553383168 logging_writer.py:48] [128000] global_step=128000, grad_norm=0.331801, loss=1.38905 +I0913 17:10:18.246575 139802006037696 submission.py:307] 128000) loss = 1.389, grad_norm = 0.332 +I0913 17:11:35.134118 139773544990464 logging_writer.py:48] [128500] global_step=128500, grad_norm=0.330005, loss=1.37361 +I0913 17:11:35.137113 139802006037696 submission.py:307] 128500) loss = 1.374, grad_norm = 0.330 +I0913 17:12:52.047034 139773553383168 logging_writer.py:48] [129000] global_step=129000, grad_norm=0.329128, loss=1.36457 +I0913 17:12:52.050259 139802006037696 submission.py:307] 129000) loss = 1.365, grad_norm = 0.329 +I0913 17:14:08.959254 139773544990464 logging_writer.py:48] [129500] global_step=129500, grad_norm=0.315518, loss=1.37255 +I0913 17:14:08.962465 139802006037696 submission.py:307] 129500) loss = 1.373, grad_norm = 0.316 +I0913 17:14:10.146197 139802006037696 spec.py:333] Evaluating on the training split. +I0913 17:14:12.346095 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 17:15:50.099857 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 17:15:52.276321 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 17:16:53.732211 139802006037696 spec.py:363] Evaluating on the test split. +I0913 17:16:55.911284 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 17:17:59.873625 139802006037696 submission_runner.py:516] Time since start: 27231.19s, Step: 129505, {'train/accuracy': 0.7083323734709392, 'train/loss': 1.285520877726969, 'train/bleu': 36.57114632263505, 'validation/accuracy': 0.6938289667828049, 'validation/loss': 1.3840151160555976, 'validation/bleu': 30.726965371385642, 'validation/num_examples': 3000, 'test/accuracy': 0.7121840683283946, 'test/loss': 1.2676883606414502, 'test/bleu': 30.779927367324674, 'test/num_examples': 3003, 'score': 19984.64667034149, 'total_duration': 27231.190740823746, 'accumulated_submission_time': 19984.64667034149, 'accumulated_eval_time': 7170.2508709430695, 'accumulated_logging_time': 1.231386423110962} +I0913 17:17:59.904602 139773553383168 logging_writer.py:48] [129505] accumulated_eval_time=7170.25, accumulated_logging_time=1.23139, accumulated_submission_time=19984.6, global_step=129505, preemption_count=0, score=19984.6, test/accuracy=0.712184, test/bleu=30.7799, test/loss=1.26769, test/num_examples=3003, total_duration=27231.2, train/accuracy=0.708332, train/bleu=36.5711, train/loss=1.28552, validation/accuracy=0.693829, validation/bleu=30.727, validation/loss=1.38402, validation/num_examples=3000 +I0913 17:19:16.566782 139773544990464 logging_writer.py:48] [130000] global_step=130000, grad_norm=0.370799, loss=1.47464 +I0913 17:19:16.569995 139802006037696 submission.py:307] 130000) loss = 1.475, grad_norm = 0.371 +I0913 17:20:33.390090 139773553383168 logging_writer.py:48] [130500] global_step=130500, grad_norm=0.350498, loss=1.40813 +I0913 17:20:33.393115 139802006037696 submission.py:307] 130500) loss = 1.408, grad_norm = 0.350 +I0913 17:21:50.290521 139773544990464 logging_writer.py:48] [131000] global_step=131000, grad_norm=0.364495, loss=1.42906 +I0913 17:21:50.293826 139802006037696 submission.py:307] 131000) loss = 1.429, grad_norm = 0.364 +I0913 17:23:07.235476 139773553383168 logging_writer.py:48] [131500] global_step=131500, grad_norm=0.343793, loss=1.40298 +I0913 17:23:07.238604 139802006037696 submission.py:307] 131500) loss = 1.403, grad_norm = 0.344 +I0913 17:24:24.196378 139773544990464 logging_writer.py:48] [132000] global_step=132000, grad_norm=0.327567, loss=1.45601 +I0913 17:24:24.199464 139802006037696 submission.py:307] 132000) loss = 1.456, grad_norm = 0.328 +I0913 17:25:41.171118 139773553383168 logging_writer.py:48] [132500] global_step=132500, grad_norm=0.342495, loss=1.38387 +I0913 17:25:41.174187 139802006037696 submission.py:307] 132500) loss = 1.384, grad_norm = 0.342 +I0913 17:26:58.165347 139773544990464 logging_writer.py:48] [133000] global_step=133000, grad_norm=0.353278, loss=1.42715 +I0913 17:26:58.168623 139802006037696 submission.py:307] 133000) loss = 1.427, grad_norm = 0.353 +I0913 17:28:15.170622 139773553383168 logging_writer.py:48] [133500] global_step=133500, grad_norm=0.329334, loss=1.41253 +I0913 17:28:15.173738 139802006037696 submission.py:307] 133500) loss = 1.413, grad_norm = 0.329 +I0913 17:28:44.529715 139802006037696 spec.py:333] Evaluating on the training split. +I0913 17:28:46.724269 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 17:30:17.562226 139802006037696 spec.py:346] Evaluating on the validation split. +I0913 17:30:19.744868 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 17:31:25.935719 139802006037696 spec.py:363] Evaluating on the test split. +I0913 17:31:28.112091 139802006037696 workload.py:152] Translating evaluation dataset. +I0913 17:32:27.622565 139802006037696 submission_runner.py:516] Time since start: 28098.94s, Step: 133688, {'train/accuracy': 0.706277744003854, 'train/loss': 1.294006327353437, 'train/bleu': 36.788203593365424, 'validation/accuracy': 0.6942381371588697, 'validation/loss': 1.3840120162800214, 'validation/bleu': 30.880696709612252, 'validation/num_examples': 3000, 'test/accuracy': 0.7111266050781477, 'test/loss': 1.2693124128464355, 'test/bleu': 30.65709660704041, 'test/num_examples': 3003, 'score': 20626.889546871185, 'total_duration': 28098.93967318535, 'accumulated_submission_time': 20626.889546871185, 'accumulated_eval_time': 7393.343722343445, 'accumulated_logging_time': 1.2717046737670898} +I0913 17:32:27.654809 139773544990464 logging_writer.py:48] [133688] accumulated_eval_time=7393.34, accumulated_logging_time=1.2717, accumulated_submission_time=20626.9, global_step=133688, preemption_count=0, score=20626.9, test/accuracy=0.711127, test/bleu=30.6571, test/loss=1.26931, test/num_examples=3003, total_duration=28098.9, train/accuracy=0.706278, train/bleu=36.7882, train/loss=1.29401, validation/accuracy=0.694238, validation/bleu=30.8807, validation/loss=1.38401, validation/num_examples=3000 +I0913 17:32:28.233704 139773553383168 logging_writer.py:48] [133688] global_step=133688, preemption_count=0, score=20626.9 +I0913 17:32:28.251448 139802006037696 submission_runner.py:857] Final wmt score: 20626.889546871185 diff --git a/logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/criteo1tb_pytorch_09-15-2026-19-27-27.log b/logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/criteo1tb_pytorch_09-15-2026-19-27-27.log new file mode 100644 index 00000000..8fdb0f66 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/criteo1tb_pytorch_09-15-2026-19-27-27.log @@ -0,0 +1,523 @@ +torchrun --redirects 1:0,2:0,3:0 --standalone --nnodes=1 --nproc_per_node=4 submission_runner.py --framework=pytorch --workload=criteo1tb --submission_path=submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py --data_dir=/data/criteo1tb --experiment_dir=/experiment_runs --experiment_name=submissions_a100/ademamix_golden/study_1 --overwrite=True --save_checkpoints=False --rng_seed=1213513687 --torch_compile=true --tuning_ruleset=self 2>&1 | tee -a /logs/criteo1tb_pytorch_09-15-2026-19-27-27.log +W0915 19:27:52.658000 9 site-packages/torch/distributed/run.py:803] +W0915 19:27:52.658000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0915 19:27:52.658000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0915 19:27:52.658000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-15 19:28:07.197125: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-15 19:28:07.197112: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-15 19:28:07.197121: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-15 19:28:07.197115: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789500487.747164 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789500487.747164 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789500487.747154 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789500487.747184 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789500487.807429 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789500487.807431 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789500487.807431 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789500487.807453 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789500489.216652 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789500489.216663 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789500489.216663 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789500489.216671 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789500489.216700 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789500489.216700 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789500489.216701 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789500489.216703 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789500489.216704 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789500489.216704 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789500489.216703 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789500489.216705 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789500489.216706 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789500489.216707 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789500489.216707 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789500489.216709 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789500516.234264 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789500516.234308 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789500516.234371 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789500516.272564 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W915 19:28:44.803700246 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0915 19:28:46.831246 139954106807488 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/criteo1tb_pytorch. +I0915 19:28:46.831248 139868064736448 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/criteo1tb_pytorch. +I0915 19:28:46.831247 140044445967552 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/criteo1tb_pytorch. +I0915 19:28:46.831269 139751210816704 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/criteo1tb_pytorch. +I0915 19:28:47.070083 139954106807488 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/criteo1tb_pytorch/trial_1. +I0915 19:28:47.335212 139954106807488 submission_runner.py:242] Initializing dataset. +I0915 19:28:47.335382 139954106807488 submission_runner.py:251] Initializing model. +W0915 19:29:00.720472 139751210816704 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0915 19:29:00.720470 140044445967552 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0915 19:29:00.720479 139868064736448 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0915 19:29:00.720507 139954106807488 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +I0915 19:29:00.720698 139954106807488 submission_runner.py:294] Initializing optimizer. +I0915 19:29:00.721112 139954106807488 submission_runner.py:299] Initializing metrics bundle. +I0915 19:29:00.721278 139954106807488 submission_runner.py:321] Initializing checkpoint and logger. +I0915 19:29:00.724020 139954106807488 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_1/criteo1tb_pytorch/trial_1/meta_data_0.json. +I0915 19:29:00.724093 139868064736448 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0915 19:29:00.724103 140044445967552 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0915 19:29:00.724134 139751210816704 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0915 19:29:00.724217 139868064736448 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0915 19:29:00.724221 140044445967552 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0915 19:29:00.724234 139954106807488 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0915 19:29:00.724269 139751210816704 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0915 19:29:00.724291 139954106807488 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0915 19:29:01.181237 139954106807488 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_1/criteo1tb_pytorch/trial_1/flags_0.json. +I0915 19:29:01.299731 139954106807488 submission_runner.py:359] Starting training loop. +I0915 19:29:10.778169 139930103424768 logging_writer.py:48] [0] global_step=0, grad_norm=0.5, loss=1.34427 +I0915 19:29:10.957363 139954106807488 submission.py:307] 0) loss = 1.344, grad_norm = 0.500 +I0915 19:29:11.274874 139954106807488 spec.py:333] Evaluating on the training split. +I0915 19:39:01.557878 139954106807488 spec.py:346] Evaluating on the validation split. +I0915 19:48:50.243792 139954106807488 spec.py:363] Evaluating on the test split. +I0915 20:00:11.955152 139954106807488 submission_runner.py:516] Time since start: 1870.63s, Step: 1, {'train/loss': 1.3445777583756644, 'validation/loss': 1.3570842317910405, 'validation/num_examples': 83274637, 'test/loss': 1.3495619400801808, 'test/num_examples': 95000000, 'score': 9.658968210220337, 'total_duration': 1870.628292798996, 'accumulated_submission_time': 9.658968210220337, 'accumulated_eval_time': 1860.652927160263, 'accumulated_logging_time': 0} +I0915 20:00:12.003309 139910849693440 logging_writer.py:48] [1] accumulated_eval_time=1860.65, accumulated_logging_time=0, accumulated_submission_time=9.65897, global_step=1, preemption_count=0, score=9.65897, test/loss=1.34956, test/num_examples=95000000, total_duration=1870.63, train/loss=1.34458, validation/loss=1.35708, validation/num_examples=83274637 +I0915 20:00:13.180682 139910841300736 logging_writer.py:48] [1] global_step=1, grad_norm=0.5, loss=1.3444 +I0915 20:00:13.183721 139954106807488 submission.py:307] 1) loss = 1.344, grad_norm = 0.500 +I0915 20:00:13.419177 139910849693440 logging_writer.py:48] [2] global_step=2, grad_norm=0.5, loss=1.33807 +I0915 20:00:13.422201 139954106807488 submission.py:307] 2) loss = 1.338, grad_norm = 0.500 +I0915 20:00:13.660265 139910841300736 logging_writer.py:48] [3] global_step=3, grad_norm=0.5, loss=1.32062 +I0915 20:00:13.663316 139954106807488 submission.py:307] 3) loss = 1.321, grad_norm = 0.500 +I0915 20:00:13.897544 139910849693440 logging_writer.py:48] [4] global_step=4, grad_norm=0.5, loss=1.29969 +I0915 20:00:13.900515 139954106807488 submission.py:307] 4) loss = 1.300, grad_norm = 0.500 +I0915 20:00:14.136694 139910841300736 logging_writer.py:48] [5] global_step=5, grad_norm=0.5, loss=1.26853 +I0915 20:00:14.139666 139954106807488 submission.py:307] 5) loss = 1.269, grad_norm = 0.500 +I0915 20:00:14.374011 139910849693440 logging_writer.py:48] [6] global_step=6, grad_norm=0.5, loss=1.2317 +I0915 20:00:14.376937 139954106807488 submission.py:307] 6) loss = 1.232, grad_norm = 0.500 +I0915 20:00:14.611555 139910841300736 logging_writer.py:48] [7] global_step=7, grad_norm=0.5, loss=1.18892 +I0915 20:00:14.614563 139954106807488 submission.py:307] 7) loss = 1.189, grad_norm = 0.500 +I0915 20:00:14.851361 139910849693440 logging_writer.py:48] [8] global_step=8, grad_norm=0.5, loss=1.1378 +I0915 20:00:14.854482 139954106807488 submission.py:307] 8) loss = 1.138, grad_norm = 0.500 +I0915 20:00:15.091191 139910841300736 logging_writer.py:48] [9] global_step=9, grad_norm=0.5, loss=1.08053 +I0915 20:00:15.094295 139954106807488 submission.py:307] 9) loss = 1.081, grad_norm = 0.500 +I0915 20:00:15.328712 139910849693440 logging_writer.py:48] [10] global_step=10, grad_norm=0.5, loss=1.01737 +I0915 20:00:15.331699 139954106807488 submission.py:307] 10) loss = 1.017, grad_norm = 0.500 +I0915 20:00:15.567795 139910841300736 logging_writer.py:48] [11] global_step=11, grad_norm=0.5, loss=0.94735 +I0915 20:00:15.570892 139954106807488 submission.py:307] 11) loss = 0.947, grad_norm = 0.500 +I0915 20:00:15.804484 139910849693440 logging_writer.py:48] [12] global_step=12, grad_norm=0.5, loss=0.873481 +I0915 20:00:15.807484 139954106807488 submission.py:307] 12) loss = 0.873, grad_norm = 0.500 +I0915 20:00:16.044326 139910841300736 logging_writer.py:48] [13] global_step=13, grad_norm=0.5, loss=0.79731 +I0915 20:00:16.047355 139954106807488 submission.py:307] 13) loss = 0.797, grad_norm = 0.500 +I0915 20:00:16.282538 139910849693440 logging_writer.py:48] [14] global_step=14, grad_norm=0.5, loss=0.720743 +I0915 20:00:16.285498 139954106807488 submission.py:307] 14) loss = 0.721, grad_norm = 0.500 +I0915 20:00:16.522302 139910841300736 logging_writer.py:48] [15] global_step=15, grad_norm=0.5, loss=0.644339 +I0915 20:00:16.525396 139954106807488 submission.py:307] 15) loss = 0.644, grad_norm = 0.500 +I0915 20:00:16.760443 139910849693440 logging_writer.py:48] [16] global_step=16, grad_norm=0.5, loss=0.568907 +I0915 20:00:16.763464 139954106807488 submission.py:307] 16) loss = 0.569, grad_norm = 0.500 +I0915 20:00:17.000788 139910841300736 logging_writer.py:48] [17] global_step=17, grad_norm=0.5, loss=0.497582 +I0915 20:00:17.003853 139954106807488 submission.py:307] 17) loss = 0.498, grad_norm = 0.500 +I0915 20:00:17.252676 139910849693440 logging_writer.py:48] [18] global_step=18, grad_norm=0.5, loss=0.431009 +I0915 20:00:17.255660 139954106807488 submission.py:307] 18) loss = 0.431, grad_norm = 0.500 +I0915 20:00:17.488158 139910841300736 logging_writer.py:48] [19] global_step=19, grad_norm=0.5, loss=0.370164 +I0915 20:00:17.491168 139954106807488 submission.py:307] 19) loss = 0.370, grad_norm = 0.500 +I0915 20:00:17.723843 139910849693440 logging_writer.py:48] [20] global_step=20, grad_norm=0.5, loss=0.32077 +I0915 20:00:17.726865 139954106807488 submission.py:307] 20) loss = 0.321, grad_norm = 0.500 +I0915 20:00:17.962052 139910841300736 logging_writer.py:48] [21] global_step=21, grad_norm=0.5, loss=0.285011 +I0915 20:00:17.965160 139954106807488 submission.py:307] 21) loss = 0.285, grad_norm = 0.500 +I0915 20:00:18.197441 139910849693440 logging_writer.py:48] [22] global_step=22, grad_norm=0.5, loss=0.252576 +I0915 20:00:18.200402 139954106807488 submission.py:307] 22) loss = 0.253, grad_norm = 0.500 +I0915 20:00:18.433627 139910841300736 logging_writer.py:48] [23] global_step=23, grad_norm=0.499999, loss=0.236988 +I0915 20:00:18.436645 139954106807488 submission.py:307] 23) loss = 0.237, grad_norm = 0.500 +I0915 20:00:18.670066 139910849693440 logging_writer.py:48] [24] global_step=24, grad_norm=0.390713, loss=0.224642 +I0915 20:00:18.673138 139954106807488 submission.py:307] 24) loss = 0.225, grad_norm = 0.391 +I0915 20:00:18.907549 139910841300736 logging_writer.py:48] [25] global_step=25, grad_norm=0.442018, loss=0.218778 +I0915 20:00:18.910687 139954106807488 submission.py:307] 25) loss = 0.219, grad_norm = 0.442 +I0915 20:00:19.144306 139910849693440 logging_writer.py:48] [26] global_step=26, grad_norm=0.499999, loss=0.223026 +I0915 20:00:19.147397 139954106807488 submission.py:307] 26) loss = 0.223, grad_norm = 0.500 +I0915 20:00:19.380154 139910841300736 logging_writer.py:48] [27] global_step=27, grad_norm=0.499999, loss=0.226404 +I0915 20:00:19.383264 139954106807488 submission.py:307] 27) loss = 0.226, grad_norm = 0.500 +I0915 20:00:19.617376 139910849693440 logging_writer.py:48] [28] global_step=28, grad_norm=0.499999, loss=0.221163 +I0915 20:00:19.620453 139954106807488 submission.py:307] 28) loss = 0.221, grad_norm = 0.500 +I0915 20:00:19.854358 139910841300736 logging_writer.py:48] [29] global_step=29, grad_norm=0.499999, loss=0.212918 +I0915 20:00:19.857378 139954106807488 submission.py:307] 29) loss = 0.213, grad_norm = 0.500 +I0915 20:00:21.851357 139910849693440 logging_writer.py:48] [30] global_step=30, grad_norm=0.499999, loss=0.202856 +I0915 20:00:21.854455 139954106807488 submission.py:307] 30) loss = 0.203, grad_norm = 0.500 +I0915 20:00:23.821696 139910841300736 logging_writer.py:48] [31] global_step=31, grad_norm=0.371407, loss=0.198789 +I0915 20:00:23.824821 139954106807488 submission.py:307] 31) loss = 0.199, grad_norm = 0.371 +I0915 20:00:25.904151 139910849693440 logging_writer.py:48] [32] global_step=32, grad_norm=0.24784, loss=0.19207 +I0915 20:00:25.907238 139954106807488 submission.py:307] 32) loss = 0.192, grad_norm = 0.248 +I0915 20:00:28.149619 139910841300736 logging_writer.py:48] [33] global_step=33, grad_norm=0.290916, loss=0.191845 +I0915 20:00:28.152758 139954106807488 submission.py:307] 33) loss = 0.192, grad_norm = 0.291 +I0915 20:00:30.323272 139910849693440 logging_writer.py:48] [34] global_step=34, grad_norm=0.302643, loss=0.188725 +I0915 20:00:30.326359 139954106807488 submission.py:307] 34) loss = 0.189, grad_norm = 0.303 +I0915 20:00:32.235526 139910841300736 logging_writer.py:48] [35] global_step=35, grad_norm=0.232693, loss=0.181207 +I0915 20:00:32.238704 139954106807488 submission.py:307] 35) loss = 0.181, grad_norm = 0.233 +I0915 20:00:34.545101 139910849693440 logging_writer.py:48] [36] global_step=36, grad_norm=0.177011, loss=0.179977 +I0915 20:00:34.548173 139954106807488 submission.py:307] 36) loss = 0.180, grad_norm = 0.177 +I0915 20:00:36.656116 139910841300736 logging_writer.py:48] [37] global_step=37, grad_norm=0.250006, loss=0.174772 +I0915 20:00:36.659165 139954106807488 submission.py:307] 37) loss = 0.175, grad_norm = 0.250 +I0915 20:00:38.530713 139910849693440 logging_writer.py:48] [38] global_step=38, grad_norm=0.18766, loss=0.170558 +I0915 20:00:38.533754 139954106807488 submission.py:307] 38) loss = 0.171, grad_norm = 0.188 +I0915 20:00:40.578042 139910841300736 logging_writer.py:48] [39] global_step=39, grad_norm=0.157715, loss=0.165008 +I0915 20:00:40.581073 139954106807488 submission.py:307] 39) loss = 0.165, grad_norm = 0.158 +I0915 20:00:42.709939 139910849693440 logging_writer.py:48] [40] global_step=40, grad_norm=0.133243, loss=0.162582 +I0915 20:00:42.712916 139954106807488 submission.py:307] 40) loss = 0.163, grad_norm = 0.133 +I0915 20:00:44.713416 139910841300736 logging_writer.py:48] [41] global_step=41, grad_norm=0.12077, loss=0.160824 +I0915 20:00:44.716441 139954106807488 submission.py:307] 41) loss = 0.161, grad_norm = 0.121 +I0915 20:00:46.577484 139910849693440 logging_writer.py:48] [42] global_step=42, grad_norm=0.104621, loss=0.157071 +I0915 20:00:46.580537 139954106807488 submission.py:307] 42) loss = 0.157, grad_norm = 0.105 +I0915 20:00:48.650842 139910841300736 logging_writer.py:48] [43] global_step=43, grad_norm=0.0982749, loss=0.154367 +I0915 20:00:48.654256 139954106807488 submission.py:307] 43) loss = 0.154, grad_norm = 0.098 +I0915 20:00:50.489950 139910849693440 logging_writer.py:48] [44] global_step=44, grad_norm=0.0907252, loss=0.149283 +I0915 20:00:50.493050 139954106807488 submission.py:307] 44) loss = 0.149, grad_norm = 0.091 +I0915 20:00:52.528755 139910841300736 logging_writer.py:48] [45] global_step=45, grad_norm=0.120149, loss=0.149617 +I0915 20:00:52.531764 139954106807488 submission.py:307] 45) loss = 0.150, grad_norm = 0.120 +I0915 20:00:54.881362 139910849693440 logging_writer.py:48] [46] global_step=46, grad_norm=0.116981, loss=0.149308 +I0915 20:00:54.884425 139954106807488 submission.py:307] 46) loss = 0.149, grad_norm = 0.117 +I0915 20:00:56.788174 139910841300736 logging_writer.py:48] [47] global_step=47, grad_norm=0.0750321, loss=0.147588 +I0915 20:00:56.791188 139954106807488 submission.py:307] 47) loss = 0.148, grad_norm = 0.075 +I0915 20:00:58.543730 139910849693440 logging_writer.py:48] [48] global_step=48, grad_norm=0.087141, loss=0.145498 +I0915 20:00:58.546782 139954106807488 submission.py:307] 48) loss = 0.145, grad_norm = 0.087 +I0915 20:01:00.596484 139910841300736 logging_writer.py:48] [49] global_step=49, grad_norm=0.0745422, loss=0.144648 +I0915 20:01:00.599559 139954106807488 submission.py:307] 49) loss = 0.145, grad_norm = 0.075 +I0915 20:01:02.923525 139910849693440 logging_writer.py:48] [50] global_step=50, grad_norm=0.102328, loss=0.144314 +I0915 20:01:02.926530 139954106807488 submission.py:307] 50) loss = 0.144, grad_norm = 0.102 +I0915 20:01:05.017790 139910841300736 logging_writer.py:48] [51] global_step=51, grad_norm=0.0892823, loss=0.142085 +I0915 20:01:05.020866 139954106807488 submission.py:307] 51) loss = 0.142, grad_norm = 0.089 +I0915 20:01:06.879462 139910849693440 logging_writer.py:48] [52] global_step=52, grad_norm=0.0588252, loss=0.138392 +I0915 20:01:06.882518 139954106807488 submission.py:307] 52) loss = 0.138, grad_norm = 0.059 +I0915 20:01:09.241557 139910841300736 logging_writer.py:48] [53] global_step=53, grad_norm=0.155145, loss=0.138671 +I0915 20:01:09.244675 139954106807488 submission.py:307] 53) loss = 0.139, grad_norm = 0.155 +I0915 20:01:11.329763 139910849693440 logging_writer.py:48] [54] global_step=54, grad_norm=0.107676, loss=0.138166 +I0915 20:01:11.332864 139954106807488 submission.py:307] 54) loss = 0.138, grad_norm = 0.108 +I0915 20:01:13.494214 139910841300736 logging_writer.py:48] [55] global_step=55, grad_norm=0.0469033, loss=0.137103 +I0915 20:01:13.497316 139954106807488 submission.py:307] 55) loss = 0.137, grad_norm = 0.047 +I0915 20:01:15.685633 139910849693440 logging_writer.py:48] [56] global_step=56, grad_norm=0.150599, loss=0.136517 +I0915 20:01:15.688696 139954106807488 submission.py:307] 56) loss = 0.137, grad_norm = 0.151 +I0915 20:01:17.546811 139910841300736 logging_writer.py:48] [57] global_step=57, grad_norm=0.185356, loss=0.136744 +I0915 20:01:17.549864 139954106807488 submission.py:307] 57) loss = 0.137, grad_norm = 0.185 +I0915 20:01:19.773429 139910849693440 logging_writer.py:48] [58] global_step=58, grad_norm=0.146681, loss=0.134828 +I0915 20:01:19.776475 139954106807488 submission.py:307] 58) loss = 0.135, grad_norm = 0.147 +I0915 20:01:21.882727 139910841300736 logging_writer.py:48] [59] global_step=59, grad_norm=0.0477874, loss=0.136687 +I0915 20:01:21.885846 139954106807488 submission.py:307] 59) loss = 0.137, grad_norm = 0.048 +I0915 20:01:23.999485 139910849693440 logging_writer.py:48] [60] global_step=60, grad_norm=0.139275, loss=0.133472 +I0915 20:01:24.002841 139954106807488 submission.py:307] 60) loss = 0.133, grad_norm = 0.139 +I0915 20:01:26.117153 139910841300736 logging_writer.py:48] [61] global_step=61, grad_norm=0.331062, loss=0.135258 +I0915 20:01:26.120274 139954106807488 submission.py:307] 61) loss = 0.135, grad_norm = 0.331 +I0915 20:01:28.289811 139910849693440 logging_writer.py:48] [62] global_step=62, grad_norm=0.499999, loss=0.1332 +I0915 20:01:28.292922 139954106807488 submission.py:307] 62) loss = 0.133, grad_norm = 0.500 +I0915 20:01:30.516544 139910841300736 logging_writer.py:48] [63] global_step=63, grad_norm=0.410707, loss=0.133623 +I0915 20:01:30.519545 139954106807488 submission.py:307] 63) loss = 0.134, grad_norm = 0.411 +I0915 20:01:32.627273 139910849693440 logging_writer.py:48] [64] global_step=64, grad_norm=0.0206294, loss=0.132403 +I0915 20:01:32.630364 139954106807488 submission.py:307] 64) loss = 0.132, grad_norm = 0.021 +I0915 20:01:34.961988 139910841300736 logging_writer.py:48] [65] global_step=65, grad_norm=0.416475, loss=0.132719 +I0915 20:01:34.965083 139954106807488 submission.py:307] 65) loss = 0.133, grad_norm = 0.416 +I0915 20:01:37.070353 139910849693440 logging_writer.py:48] [66] global_step=66, grad_norm=0.454462, loss=0.133467 +I0915 20:01:37.073471 139954106807488 submission.py:307] 66) loss = 0.133, grad_norm = 0.454 +I0915 20:01:38.969537 139910841300736 logging_writer.py:48] [67] global_step=67, grad_norm=0.0254821, loss=0.132092 +I0915 20:01:38.972601 139954106807488 submission.py:307] 67) loss = 0.132, grad_norm = 0.025 +I0915 20:01:41.054769 139910849693440 logging_writer.py:48] [68] global_step=68, grad_norm=0.446659, loss=0.133774 +I0915 20:01:41.057808 139954106807488 submission.py:307] 68) loss = 0.134, grad_norm = 0.447 +I0915 20:01:43.196160 139910841300736 logging_writer.py:48] [69] global_step=69, grad_norm=0.397115, loss=0.131018 +I0915 20:01:43.199201 139954106807488 submission.py:307] 69) loss = 0.131, grad_norm = 0.397 +I0915 20:01:45.277044 139910849693440 logging_writer.py:48] [70] global_step=70, grad_norm=0.121745, loss=0.131279 +I0915 20:01:45.280047 139954106807488 submission.py:307] 70) loss = 0.131, grad_norm = 0.122 +I0915 20:01:47.381686 139910841300736 logging_writer.py:48] [71] global_step=71, grad_norm=0.499999, loss=0.132862 +I0915 20:01:47.384796 139954106807488 submission.py:307] 71) loss = 0.133, grad_norm = 0.500 +I0915 20:01:49.484813 139910849693440 logging_writer.py:48] [72] global_step=72, grad_norm=0.364671, loss=0.13328 +I0915 20:01:49.488131 139954106807488 submission.py:307] 72) loss = 0.133, grad_norm = 0.365 +I0915 20:01:51.609153 139910841300736 logging_writer.py:48] [73] global_step=73, grad_norm=0.11777, loss=0.131091 +I0915 20:01:51.612163 139954106807488 submission.py:307] 73) loss = 0.131, grad_norm = 0.118 +I0915 20:01:53.453272 139910849693440 logging_writer.py:48] [74] global_step=74, grad_norm=0.443358, loss=0.132302 +I0915 20:01:53.456317 139954106807488 submission.py:307] 74) loss = 0.132, grad_norm = 0.443 +I0915 20:01:55.508916 139910841300736 logging_writer.py:48] [75] global_step=75, grad_norm=0.380214, loss=0.134778 +I0915 20:01:55.512092 139954106807488 submission.py:307] 75) loss = 0.135, grad_norm = 0.380 +I0915 20:01:57.740187 139910849693440 logging_writer.py:48] [76] global_step=76, grad_norm=0.0600906, loss=0.136137 +I0915 20:01:57.743229 139954106807488 submission.py:307] 76) loss = 0.136, grad_norm = 0.060 +I0915 20:02:00.018419 139910841300736 logging_writer.py:48] [77] global_step=77, grad_norm=0.475492, loss=0.138261 +I0915 20:02:00.021486 139954106807488 submission.py:307] 77) loss = 0.138, grad_norm = 0.475 +I0915 20:02:01.904706 139910849693440 logging_writer.py:48] [78] global_step=78, grad_norm=0.480864, loss=0.140331 +I0915 20:02:01.908130 139954106807488 submission.py:307] 78) loss = 0.140, grad_norm = 0.481 +I0915 20:02:04.093120 139910841300736 logging_writer.py:48] [79] global_step=79, grad_norm=0.0413328, loss=0.139891 +I0915 20:02:04.096100 139954106807488 submission.py:307] 79) loss = 0.140, grad_norm = 0.041 +I0915 20:02:06.136270 139910849693440 logging_writer.py:48] [80] global_step=80, grad_norm=0.455155, loss=0.14161 +I0915 20:02:06.139309 139954106807488 submission.py:307] 80) loss = 0.142, grad_norm = 0.455 +I0915 20:02:08.105386 139910841300736 logging_writer.py:48] [81] global_step=81, grad_norm=0.465209, loss=0.139112 +I0915 20:02:08.108350 139954106807488 submission.py:307] 81) loss = 0.139, grad_norm = 0.465 +I0915 20:02:10.262315 139910849693440 logging_writer.py:48] [82] global_step=82, grad_norm=0.0405145, loss=0.138972 +I0915 20:02:10.265306 139954106807488 submission.py:307] 82) loss = 0.139, grad_norm = 0.041 +I0915 20:02:12.242578 139910841300736 logging_writer.py:48] [83] global_step=83, grad_norm=0.499999, loss=0.137475 +I0915 20:02:12.245572 139954106807488 submission.py:307] 83) loss = 0.137, grad_norm = 0.500 +I0915 20:02:14.541324 139910849693440 logging_writer.py:48] [84] global_step=84, grad_norm=0.406188, loss=0.138384 +I0915 20:02:14.544313 139954106807488 submission.py:307] 84) loss = 0.138, grad_norm = 0.406 +I0915 20:02:16.339430 139910841300736 logging_writer.py:48] [85] global_step=85, grad_norm=0.109017, loss=0.136887 +I0915 20:02:16.342435 139954106807488 submission.py:307] 85) loss = 0.137, grad_norm = 0.109 +I0915 20:02:18.483224 139910849693440 logging_writer.py:48] [86] global_step=86, grad_norm=0.499999, loss=0.138066 +I0915 20:02:18.486344 139954106807488 submission.py:307] 86) loss = 0.138, grad_norm = 0.500 +I0915 20:02:20.479923 139910841300736 logging_writer.py:48] [87] global_step=87, grad_norm=0.388794, loss=0.136583 +I0915 20:02:20.483048 139954106807488 submission.py:307] 87) loss = 0.137, grad_norm = 0.389 +I0915 20:02:22.182068 139910849693440 logging_writer.py:48] [88] global_step=88, grad_norm=0.123914, loss=0.135955 +I0915 20:02:22.185155 139954106807488 submission.py:307] 88) loss = 0.136, grad_norm = 0.124 +I0915 20:02:24.246409 139910841300736 logging_writer.py:48] [89] global_step=89, grad_norm=0.499999, loss=0.137438 +I0915 20:02:24.249548 139954106807488 submission.py:307] 89) loss = 0.137, grad_norm = 0.500 +I0915 20:02:26.432579 139910849693440 logging_writer.py:48] [90] global_step=90, grad_norm=0.34474, loss=0.134689 +I0915 20:02:26.435601 139954106807488 submission.py:307] 90) loss = 0.135, grad_norm = 0.345 +I0915 20:02:28.559903 139910841300736 logging_writer.py:48] [91] global_step=91, grad_norm=0.164735, loss=0.134932 +I0915 20:02:28.562963 139954106807488 submission.py:307] 91) loss = 0.135, grad_norm = 0.165 +I0915 20:02:30.569703 139910849693440 logging_writer.py:48] [92] global_step=92, grad_norm=0.499999, loss=0.135207 +I0915 20:02:30.572741 139954106807488 submission.py:307] 92) loss = 0.135, grad_norm = 0.500 +I0915 20:02:32.501168 139910841300736 logging_writer.py:48] [93] global_step=93, grad_norm=0.362913, loss=0.136035 +I0915 20:02:32.504206 139954106807488 submission.py:307] 93) loss = 0.136, grad_norm = 0.363 +I0915 20:02:34.724279 139910849693440 logging_writer.py:48] [94] global_step=94, grad_norm=0.0733061, loss=0.133714 +I0915 20:02:34.727388 139954106807488 submission.py:307] 94) loss = 0.134, grad_norm = 0.073 +I0915 20:02:36.590569 139910841300736 logging_writer.py:48] [95] global_step=95, grad_norm=0.476824, loss=0.131824 +I0915 20:02:36.594145 139954106807488 submission.py:307] 95) loss = 0.132, grad_norm = 0.477 +I0915 20:02:38.571694 139910849693440 logging_writer.py:48] [96] global_step=96, grad_norm=0.411148, loss=0.129903 +I0915 20:02:38.574717 139954106807488 submission.py:307] 96) loss = 0.130, grad_norm = 0.411 +I0915 20:02:40.541096 139910841300736 logging_writer.py:48] [97] global_step=97, grad_norm=0.0902777, loss=0.128171 +I0915 20:02:40.544434 139954106807488 submission.py:307] 97) loss = 0.128, grad_norm = 0.090 +I0915 20:02:42.571749 139910849693440 logging_writer.py:48] [98] global_step=98, grad_norm=0.434736, loss=0.131162 +I0915 20:02:42.574713 139954106807488 submission.py:307] 98) loss = 0.131, grad_norm = 0.435 +I0915 20:02:44.507843 139910841300736 logging_writer.py:48] [99] global_step=99, grad_norm=0.299891, loss=0.130035 +I0915 20:02:44.510845 139954106807488 submission.py:307] 99) loss = 0.130, grad_norm = 0.300 +I0915 20:02:46.613850 139910849693440 logging_writer.py:48] [100] global_step=100, grad_norm=0.102488, loss=0.129593 +I0915 20:02:46.616871 139954106807488 submission.py:307] 100) loss = 0.130, grad_norm = 0.102 +I0915 20:06:09.638692 139954106807488 spec.py:333] Evaluating on the training split. +I0915 20:15:24.274420 139954106807488 spec.py:346] Evaluating on the validation split. +I0915 20:19:15.338068 139954106807488 spec.py:363] Evaluating on the test split. +I0915 20:23:42.903936 139954106807488 submission_runner.py:516] Time since start: 3281.60s, Step: 200, {'train/loss': 0.12686944608643472, 'validation/loss': 0.12899851849719382, 'validation/num_examples': 83274637, 'test/loss': 0.13144157489993447, 'test/num_examples': 95000000, 'score': 366.55974078178406, 'total_duration': 3281.604463815689, 'accumulated_submission_time': 366.55974078178406, 'accumulated_eval_time': 2913.9182076454163, 'accumulated_logging_time': 0.056310176849365234} +I0915 20:23:42.928286 139910841300736 logging_writer.py:48] [200] accumulated_eval_time=2913.92, accumulated_logging_time=0.0563102, accumulated_submission_time=366.56, global_step=200, preemption_count=0, score=366.56, test/loss=0.131442, test/num_examples=95000000, total_duration=3281.6, train/loss=0.126869, validation/loss=0.128999, validation/num_examples=83274637 +I0915 20:29:39.976366 139954106807488 spec.py:333] Evaluating on the training split. +I0915 20:38:49.402167 139954106807488 spec.py:346] Evaluating on the validation split. +I0915 20:42:41.536019 139954106807488 spec.py:363] Evaluating on the test split. +I0915 20:47:09.263425 139954106807488 submission_runner.py:516] Time since start: 4687.96s, Step: 402, {'train/loss': 0.12923112732345268, 'validation/loss': 0.12797835231330296, 'validation/num_examples': 83274637, 'test/loss': 0.13044068226430794, 'test/num_examples': 95000000, 'score': 722.8921821117401, 'total_duration': 4687.963960409164, 'accumulated_submission_time': 722.8921821117401, 'accumulated_eval_time': 3963.20534324646, 'accumulated_logging_time': 0.08729052543640137} +I0915 20:47:09.286659 139910849693440 logging_writer.py:48] [402] accumulated_eval_time=3963.21, accumulated_logging_time=0.0872905, accumulated_submission_time=722.892, global_step=402, preemption_count=0, score=722.892, test/loss=0.130441, test/num_examples=95000000, total_duration=4687.96, train/loss=0.129231, validation/loss=0.127978, validation/num_examples=83274637 +I0915 20:49:40.130976 139910841300736 logging_writer.py:48] [500] global_step=500, grad_norm=0.0453804, loss=0.12807 +I0915 20:49:40.134367 139954106807488 submission.py:307] 500) loss = 0.128, grad_norm = 0.045 +I0915 20:53:06.336317 139954106807488 spec.py:333] Evaluating on the training split. +I0915 21:02:12.105314 139954106807488 spec.py:346] Evaluating on the validation split. +I0915 21:06:03.814789 139954106807488 spec.py:363] Evaluating on the test split. +I0915 21:10:31.542563 139954106807488 submission_runner.py:516] Time since start: 6090.24s, Step: 601, {'train/loss': 0.12846036074903305, 'validation/loss': 0.12678148764953734, 'validation/num_examples': 83274637, 'test/loss': 0.12913297464583548, 'test/num_examples': 95000000, 'score': 1079.2291326522827, 'total_duration': 6090.243111848831, 'accumulated_submission_time': 1079.2291326522827, 'accumulated_eval_time': 5008.411679267883, 'accumulated_logging_time': 0.1173408031463623} +I0915 21:10:31.565266 139910849693440 logging_writer.py:48] [601] accumulated_eval_time=5008.41, accumulated_logging_time=0.117341, accumulated_submission_time=1079.23, global_step=601, preemption_count=0, score=1079.23, test/loss=0.129133, test/num_examples=95000000, total_duration=6090.24, train/loss=0.12846, validation/loss=0.126781, validation/num_examples=83274637 +I0915 21:16:28.567254 139954106807488 spec.py:333] Evaluating on the training split. +I0915 21:25:54.849238 139954106807488 spec.py:346] Evaluating on the validation split. +I0915 21:29:46.235125 139954106807488 spec.py:363] Evaluating on the test split. +I0915 21:34:13.324264 139954106807488 submission_runner.py:516] Time since start: 7512.02s, Step: 802, {'train/loss': 0.12615643360582587, 'validation/loss': 0.12629461688197174, 'validation/num_examples': 83274637, 'test/loss': 0.12872399193227668, 'test/num_examples': 95000000, 'score': 1435.5280559062958, 'total_duration': 7512.024815559387, 'accumulated_submission_time': 1435.5280559062958, 'accumulated_eval_time': 6073.168746471405, 'accumulated_logging_time': 0.1463634967803955} +I0915 21:34:13.345931 139910841300736 logging_writer.py:48] [802] accumulated_eval_time=6073.17, accumulated_logging_time=0.146363, accumulated_submission_time=1435.53, global_step=802, preemption_count=0, score=1435.53, test/loss=0.128724, test/num_examples=95000000, total_duration=7512.02, train/loss=0.126156, validation/loss=0.126295, validation/num_examples=83274637 +I0915 21:39:56.148063 139910849693440 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.017373, loss=0.121045 +I0915 21:39:56.151256 139954106807488 submission.py:307] 1000) loss = 0.121, grad_norm = 0.017 +I0915 21:40:10.665159 139954106807488 spec.py:333] Evaluating on the training split. +I0915 21:49:18.645221 139954106807488 spec.py:346] Evaluating on the validation split. +I0915 21:52:54.732950 139954106807488 spec.py:363] Evaluating on the test split. +I0915 21:57:19.464311 139954106807488 submission_runner.py:516] Time since start: 8898.16s, Step: 1008, {'train/loss': 0.12736491135558098, 'validation/loss': 0.126224402964769, 'validation/num_examples': 83274637, 'test/loss': 0.12857006342420077, 'test/num_examples': 95000000, 'score': 1792.1379549503326, 'total_duration': 8898.164870262146, 'accumulated_submission_time': 1792.1379549503326, 'accumulated_eval_time': 7101.967989206314, 'accumulated_logging_time': 0.1746838092803955} +I0915 21:57:19.486768 139910841300736 logging_writer.py:48] [1008] accumulated_eval_time=7101.97, accumulated_logging_time=0.174684, accumulated_submission_time=1792.14, global_step=1008, preemption_count=0, score=1792.14, test/loss=0.12857, test/num_examples=95000000, total_duration=8898.16, train/loss=0.127365, validation/loss=0.126224, validation/num_examples=83274637 +I0915 22:03:16.402532 139954106807488 spec.py:333] Evaluating on the training split. +I0915 22:12:17.397317 139954106807488 spec.py:346] Evaluating on the validation split. +I0915 22:15:52.425201 139954106807488 spec.py:363] Evaluating on the test split. +I0915 22:20:17.684293 139954106807488 submission_runner.py:516] Time since start: 10276.38s, Step: 1220, {'train/loss': 0.12496987565034151, 'validation/loss': 0.12579652161119198, 'validation/num_examples': 83274637, 'test/loss': 0.1281569895374499, 'test/num_examples': 95000000, 'score': 2148.3401172161102, 'total_duration': 10276.384850740433, 'accumulated_submission_time': 2148.3401172161102, 'accumulated_eval_time': 8123.249813318253, 'accumulated_logging_time': 0.2063307762145996} +I0915 22:20:17.706629 139910849693440 logging_writer.py:48] [1220] accumulated_eval_time=8123.25, accumulated_logging_time=0.206331, accumulated_submission_time=2148.34, global_step=1220, preemption_count=0, score=2148.34, test/loss=0.128157, test/num_examples=95000000, total_duration=10276.4, train/loss=0.12497, validation/loss=0.125797, validation/num_examples=83274637 +I0915 22:26:14.389634 139954106807488 spec.py:333] Evaluating on the training split. +I0915 22:35:09.902994 139954106807488 spec.py:346] Evaluating on the validation split. +I0915 22:38:43.186836 139954106807488 spec.py:363] Evaluating on the test split. +I0915 22:43:05.221144 139954106807488 submission_runner.py:516] Time since start: 11643.92s, Step: 1434, {'train/loss': 0.12629988249875027, 'validation/loss': 0.12569706996930202, 'validation/num_examples': 83274637, 'test/loss': 0.1278985934318141, 'test/num_examples': 95000000, 'score': 2504.3137986660004, 'total_duration': 11643.921679496765, 'accumulated_submission_time': 2504.3137986660004, 'accumulated_eval_time': 9134.081369876862, 'accumulated_logging_time': 0.23543143272399902} +I0915 22:43:05.244324 139910841300736 logging_writer.py:48] [1434] accumulated_eval_time=9134.08, accumulated_logging_time=0.235431, accumulated_submission_time=2504.31, global_step=1434, preemption_count=0, score=2504.31, test/loss=0.127899, test/num_examples=95000000, total_duration=11643.9, train/loss=0.1263, validation/loss=0.125697, validation/num_examples=83274637 +I0915 22:44:29.792066 139910849693440 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.0497594, loss=0.127844 +I0915 22:44:29.795058 139954106807488 submission.py:307] 1500) loss = 0.128, grad_norm = 0.050 +I0915 22:49:02.442855 139954106807488 spec.py:333] Evaluating on the training split. +I0915 22:57:34.574441 139954106807488 spec.py:346] Evaluating on the validation split. +I0915 23:01:05.852151 139954106807488 spec.py:363] Evaluating on the test split. +I0915 23:05:23.988765 139954106807488 submission_runner.py:516] Time since start: 12982.69s, Step: 1636, {'train/loss': 0.12364818408796514, 'validation/loss': 0.12559302556763283, 'validation/num_examples': 83274637, 'test/loss': 0.12791898632025467, 'test/num_examples': 95000000, 'score': 2860.8038444519043, 'total_duration': 12982.689307451248, 'accumulated_submission_time': 2860.8038444519043, 'accumulated_eval_time': 10115.627338647842, 'accumulated_logging_time': 0.26520299911499023} +I0915 23:05:24.011240 139910841300736 logging_writer.py:48] [1636] accumulated_eval_time=10115.6, accumulated_logging_time=0.265203, accumulated_submission_time=2860.8, global_step=1636, preemption_count=0, score=2860.8, test/loss=0.127919, test/num_examples=95000000, total_duration=12982.7, train/loss=0.123648, validation/loss=0.125593, validation/num_examples=83274637 +I0915 23:11:21.668761 139954106807488 spec.py:333] Evaluating on the training split. +I0915 23:19:36.292882 139954106807488 spec.py:346] Evaluating on the validation split. +I0915 23:23:03.478832 139954106807488 spec.py:363] Evaluating on the test split. +I0915 23:27:00.745684 139954106807488 submission_runner.py:516] Time since start: 14279.45s, Step: 1846, {'train/loss': 0.12256910663538564, 'validation/loss': 0.12554404219358498, 'validation/num_examples': 83274637, 'test/loss': 0.12794173681351512, 'test/num_examples': 95000000, 'score': 3217.747561454773, 'total_duration': 14279.446210861206, 'accumulated_submission_time': 3217.747561454773, 'accumulated_eval_time': 11054.704303264618, 'accumulated_logging_time': 0.2939579486846924} +I0915 23:27:00.768395 139910849693440 logging_writer.py:48] [1846] accumulated_eval_time=11054.7, accumulated_logging_time=0.293958, accumulated_submission_time=3217.75, global_step=1846, preemption_count=0, score=3217.75, test/loss=0.127942, test/num_examples=95000000, total_duration=14279.4, train/loss=0.122569, validation/loss=0.125544, validation/num_examples=83274637 +I0915 23:31:21.298362 139910841300736 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.0585377, loss=0.121602 +I0915 23:31:21.301581 139954106807488 submission.py:307] 2000) loss = 0.122, grad_norm = 0.059 +I0915 23:32:57.352880 139954106807488 spec.py:333] Evaluating on the training split. +I0915 23:40:55.634853 139954106807488 spec.py:346] Evaluating on the validation split. +I0915 23:44:13.519577 139954106807488 spec.py:363] Evaluating on the test split. +I0915 23:48:02.180939 139954106807488 submission_runner.py:516] Time since start: 15540.88s, Step: 2059, {'train/loss': 0.12354772736065744, 'validation/loss': 0.12530044522672135, 'validation/num_examples': 83274637, 'test/loss': 0.12782435105895995, 'test/num_examples': 95000000, 'score': 3573.622041463852, 'total_duration': 15540.881484270096, 'accumulated_submission_time': 3573.622041463852, 'accumulated_eval_time': 11959.5324447155, 'accumulated_logging_time': 0.32340550422668457} +I0915 23:48:02.204767 139910849693440 logging_writer.py:48] [2059] accumulated_eval_time=11959.5, accumulated_logging_time=0.323406, accumulated_submission_time=3573.62, global_step=2059, preemption_count=0, score=3573.62, test/loss=0.127824, test/num_examples=95000000, total_duration=15540.9, train/loss=0.123548, validation/loss=0.1253, validation/num_examples=83274637 +I0915 23:53:58.545754 139954106807488 spec.py:333] Evaluating on the training split. +I0916 00:00:45.972218 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 00:03:43.834305 139954106807488 spec.py:363] Evaluating on the test split. +I0916 00:07:32.273742 139954106807488 submission_runner.py:516] Time since start: 16710.97s, Step: 2264, {'train/loss': 0.12145907139490257, 'validation/loss': 0.12513493672739248, 'validation/num_examples': 83274637, 'test/loss': 0.1276238236686305, 'test/num_examples': 95000000, 'score': 3929.251423597336, 'total_duration': 16710.97428059578, 'accumulated_submission_time': 3929.251423597336, 'accumulated_eval_time': 12773.260489463806, 'accumulated_logging_time': 0.3537471294403076} +I0916 00:07:32.297275 139910841300736 logging_writer.py:48] [2264] accumulated_eval_time=12773.3, accumulated_logging_time=0.353747, accumulated_submission_time=3929.25, global_step=2264, preemption_count=0, score=3929.25, test/loss=0.127624, test/num_examples=95000000, total_duration=16711, train/loss=0.121459, validation/loss=0.125135, validation/num_examples=83274637 +I0916 00:13:29.420077 139954106807488 spec.py:333] Evaluating on the training split. +I0916 00:18:52.108420 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 00:21:50.411141 139954106807488 spec.py:363] Evaluating on the test split. +I0916 00:25:49.354166 139954106807488 submission_runner.py:516] Time since start: 17808.05s, Step: 2470, {'train/loss': 0.12351140907166565, 'validation/loss': 0.1252825022753245, 'validation/num_examples': 83274637, 'test/loss': 0.12774588727031508, 'test/num_examples': 95000000, 'score': 4285.6617822647095, 'total_duration': 17808.054710149765, 'accumulated_submission_time': 4285.6617822647095, 'accumulated_eval_time': 13513.194631576538, 'accumulated_logging_time': 0.3836984634399414} +I0916 00:25:49.375616 139910849693440 logging_writer.py:48] [2470] accumulated_eval_time=13513.2, accumulated_logging_time=0.383698, accumulated_submission_time=4285.66, global_step=2470, preemption_count=0, score=4285.66, test/loss=0.127746, test/num_examples=95000000, total_duration=17808.1, train/loss=0.123511, validation/loss=0.125283, validation/num_examples=83274637 +I0916 00:25:56.676270 139910841300736 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.00709997, loss=0.116849 +I0916 00:25:56.679209 139954106807488 submission.py:307] 2500) loss = 0.117, grad_norm = 0.007 +I0916 00:31:47.126991 139954106807488 spec.py:333] Evaluating on the training split. +I0916 00:35:43.206616 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 00:38:40.241631 139954106807488 spec.py:363] Evaluating on the test split. +I0916 00:42:45.227618 139954106807488 submission_runner.py:516] Time since start: 18823.93s, Step: 2695, {'train/loss': 0.1228290405714404, 'validation/loss': 0.12496271531906078, 'validation/num_examples': 83274637, 'test/loss': 0.12739505892141242, 'test/num_examples': 95000000, 'score': 4642.696635723114, 'total_duration': 18823.92814230919, 'accumulated_submission_time': 4642.696635723114, 'accumulated_eval_time': 14171.295335292816, 'accumulated_logging_time': 0.41157078742980957} +I0916 00:42:45.251604 139910849693440 logging_writer.py:48] [2695] accumulated_eval_time=14171.3, accumulated_logging_time=0.411571, accumulated_submission_time=4642.7, global_step=2695, preemption_count=0, score=4642.7, test/loss=0.127395, test/num_examples=95000000, total_duration=18823.9, train/loss=0.122829, validation/loss=0.124963, validation/num_examples=83274637 +I0916 00:48:41.567984 139954106807488 spec.py:333] Evaluating on the training split. +I0916 00:50:12.226345 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 00:53:10.165740 139954106807488 spec.py:363] Evaluating on the test split. +I0916 00:57:15.859548 139954106807488 submission_runner.py:516] Time since start: 19694.56s, Step: 2915, {'train/loss': 0.12457669032790747, 'validation/loss': 0.12476028311265129, 'validation/num_examples': 83274637, 'test/loss': 0.12726921461808055, 'test/num_examples': 95000000, 'score': 4998.295198917389, 'total_duration': 19694.56010222435, 'accumulated_submission_time': 4998.295198917389, 'accumulated_eval_time': 14685.586983680725, 'accumulated_logging_time': 0.44234395027160645} +I0916 00:57:15.883657 139910841300736 logging_writer.py:48] [2915] accumulated_eval_time=14685.6, accumulated_logging_time=0.442344, accumulated_submission_time=4998.3, global_step=2915, preemption_count=0, score=4998.3, test/loss=0.127269, test/num_examples=95000000, total_duration=19694.6, train/loss=0.124577, validation/loss=0.12476, validation/num_examples=83274637 +I0916 00:59:15.710217 139910849693440 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.00769041, loss=0.124982 +I0916 00:59:15.713433 139954106807488 submission.py:307] 3000) loss = 0.125, grad_norm = 0.008 +I0916 01:03:13.239046 139954106807488 spec.py:333] Evaluating on the training split. +I0916 01:04:12.473790 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 01:07:09.599925 139954106807488 spec.py:363] Evaluating on the test split. +I0916 01:11:16.424979 139954106807488 submission_runner.py:516] Time since start: 20535.13s, Step: 3144, {'train/loss': 0.1250313014233972, 'validation/loss': 0.12474206268480638, 'validation/num_examples': 83274637, 'test/loss': 0.12726913738290888, 'test/num_examples': 95000000, 'score': 5354.927243947983, 'total_duration': 20535.125536441803, 'accumulated_submission_time': 5354.927243947983, 'accumulated_eval_time': 15168.773005723953, 'accumulated_logging_time': 0.4729907512664795} +I0916 01:11:16.451179 139910841300736 logging_writer.py:48] [3144] accumulated_eval_time=15168.8, accumulated_logging_time=0.472991, accumulated_submission_time=5354.93, global_step=3144, preemption_count=0, score=5354.93, test/loss=0.127269, test/num_examples=95000000, total_duration=20535.1, train/loss=0.125031, validation/loss=0.124742, validation/num_examples=83274637 +I0916 01:17:12.768772 139954106807488 spec.py:333] Evaluating on the training split. +I0916 01:18:11.780163 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 01:21:09.218853 139954106807488 spec.py:363] Evaluating on the test split. +I0916 01:25:15.558092 139954106807488 submission_runner.py:516] Time since start: 21374.26s, Step: 3366, {'train/loss': 0.12352648760409582, 'validation/loss': 0.12489402057993244, 'validation/num_examples': 83274637, 'test/loss': 0.12717002483857806, 'test/num_examples': 95000000, 'score': 5710.522954463959, 'total_duration': 21374.258639097214, 'accumulated_submission_time': 5710.522954463959, 'accumulated_eval_time': 15651.562380552292, 'accumulated_logging_time': 0.5058600902557373} +I0916 01:25:15.581366 139910849693440 logging_writer.py:48] [3366] accumulated_eval_time=15651.6, accumulated_logging_time=0.50586, accumulated_submission_time=5710.52, global_step=3366, preemption_count=0, score=5710.52, test/loss=0.12717, test/num_examples=95000000, total_duration=21374.3, train/loss=0.123526, validation/loss=0.124894, validation/num_examples=83274637 +I0916 01:28:24.736372 139910841300736 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.00612072, loss=0.113878 +I0916 01:28:24.739601 139954106807488 submission.py:307] 3500) loss = 0.114, grad_norm = 0.006 +I0916 01:31:12.152796 139954106807488 spec.py:333] Evaluating on the training split. +I0916 01:32:11.177995 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 01:35:08.933703 139954106807488 spec.py:363] Evaluating on the test split. +I0916 01:39:16.398890 139954106807488 submission_runner.py:516] Time since start: 22215.10s, Step: 3588, {'train/loss': 0.12221099068462914, 'validation/loss': 0.12477229977575673, 'validation/num_examples': 83274637, 'test/loss': 0.12706287485881604, 'test/num_examples': 95000000, 'score': 6066.370519399643, 'total_duration': 22215.099447250366, 'accumulated_submission_time': 6066.370519399643, 'accumulated_eval_time': 16135.808561325073, 'accumulated_logging_time': 0.5355989933013916} +I0916 01:39:16.421976 139910849693440 logging_writer.py:48] [3588] accumulated_eval_time=16135.8, accumulated_logging_time=0.535599, accumulated_submission_time=6066.37, global_step=3588, preemption_count=0, score=6066.37, test/loss=0.127063, test/num_examples=95000000, total_duration=22215.1, train/loss=0.122211, validation/loss=0.124772, validation/num_examples=83274637 +I0916 01:45:13.575877 139954106807488 spec.py:333] Evaluating on the training split. +I0916 01:46:12.497159 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 01:49:10.044254 139954106807488 spec.py:363] Evaluating on the test split. +I0916 01:53:16.181886 139954106807488 submission_runner.py:516] Time since start: 23054.88s, Step: 3792, {'train/loss': 0.12433552815660046, 'validation/loss': 0.12451742472973443, 'validation/num_examples': 83274637, 'test/loss': 0.126931378600753, 'test/num_examples': 95000000, 'score': 6422.8106808662415, 'total_duration': 23054.882437944412, 'accumulated_submission_time': 6422.8106808662415, 'accumulated_eval_time': 16618.414626598358, 'accumulated_logging_time': 0.5651638507843018} +I0916 01:53:16.204516 139910841300736 logging_writer.py:48] [3792] accumulated_eval_time=16618.4, accumulated_logging_time=0.565164, accumulated_submission_time=6422.81, global_step=3792, preemption_count=0, score=6422.81, test/loss=0.126931, test/num_examples=95000000, total_duration=23054.9, train/loss=0.124336, validation/loss=0.124517, validation/num_examples=83274637 +I0916 01:59:14.130293 139954106807488 spec.py:333] Evaluating on the training split. +I0916 02:00:13.175423 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 02:03:10.502968 139954106807488 spec.py:363] Evaluating on the test split. +I0916 02:07:18.513114 139954106807488 submission_runner.py:516] Time since start: 23897.21s, Step: 3988, {'train/loss': 0.12145010129290815, 'validation/loss': 0.12459337733953244, 'validation/num_examples': 83274637, 'test/loss': 0.12696915144187526, 'test/num_examples': 95000000, 'score': 6780.0280385017395, 'total_duration': 23897.213665246964, 'accumulated_submission_time': 6780.0280385017395, 'accumulated_eval_time': 17102.797516345978, 'accumulated_logging_time': 0.5944151878356934} +I0916 02:07:18.536645 139910849693440 logging_writer.py:48] [3988] accumulated_eval_time=17102.8, accumulated_logging_time=0.594415, accumulated_submission_time=6780.03, global_step=3988, preemption_count=0, score=6780.03, test/loss=0.126969, test/num_examples=95000000, total_duration=23897.2, train/loss=0.12145, validation/loss=0.124593, validation/num_examples=83274637 +I0916 02:07:21.816053 139910841300736 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.00842768, loss=0.126094 +I0916 02:07:21.819346 139954106807488 submission.py:307] 4000) loss = 0.126, grad_norm = 0.008 +I0916 02:13:15.109344 139954106807488 spec.py:333] Evaluating on the training split. +I0916 02:14:13.927597 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 02:17:10.988197 139954106807488 spec.py:363] Evaluating on the test split. +I0916 02:21:17.492530 139954106807488 submission_runner.py:516] Time since start: 24736.19s, Step: 4189, {'train/loss': 0.12253452513858708, 'validation/loss': 0.12448670608560994, 'validation/num_examples': 83274637, 'test/loss': 0.12675081651514958, 'test/num_examples': 95000000, 'score': 7135.8886506557465, 'total_duration': 24736.193083524704, 'accumulated_submission_time': 7135.8886506557465, 'accumulated_eval_time': 17585.180767774582, 'accumulated_logging_time': 0.6244139671325684} +I0916 02:21:17.515845 139910849693440 logging_writer.py:48] [4189] accumulated_eval_time=17585.2, accumulated_logging_time=0.624414, accumulated_submission_time=7135.89, global_step=4189, preemption_count=0, score=7135.89, test/loss=0.126751, test/num_examples=95000000, total_duration=24736.2, train/loss=0.122535, validation/loss=0.124487, validation/num_examples=83274637 +I0916 02:27:15.625508 139954106807488 spec.py:333] Evaluating on the training split. +I0916 02:28:14.451395 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 02:31:11.666972 139954106807488 spec.py:363] Evaluating on the test split. +I0916 02:35:18.051645 139954106807488 submission_runner.py:516] Time since start: 25576.75s, Step: 4386, {'train/loss': 0.12288636875859389, 'validation/loss': 0.12451265883311266, 'validation/num_examples': 83274637, 'test/loss': 0.12674068756344445, 'test/num_examples': 95000000, 'score': 7493.290529727936, 'total_duration': 25576.752193450928, 'accumulated_submission_time': 7493.290529727936, 'accumulated_eval_time': 18067.606957674026, 'accumulated_logging_time': 0.6540706157684326} +I0916 02:35:18.075021 139910841300736 logging_writer.py:48] [4386] accumulated_eval_time=18067.6, accumulated_logging_time=0.654071, accumulated_submission_time=7493.29, global_step=4386, preemption_count=0, score=7493.29, test/loss=0.126741, test/num_examples=95000000, total_duration=25576.8, train/loss=0.122886, validation/loss=0.124513, validation/num_examples=83274637 +I0916 02:38:26.678851 139910849693440 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.0130801, loss=0.123421 +I0916 02:38:26.682167 139954106807488 submission.py:307] 4500) loss = 0.123, grad_norm = 0.013 +I0916 02:41:16.291760 139954106807488 spec.py:333] Evaluating on the training split. +I0916 02:42:15.285953 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 02:45:13.538394 139954106807488 spec.py:363] Evaluating on the test split. +I0916 02:49:21.915838 139954106807488 submission_runner.py:516] Time since start: 26420.62s, Step: 4597, {'train/loss': 0.12418175615610662, 'validation/loss': 0.12458613103884075, 'validation/num_examples': 83274637, 'test/loss': 0.12681855282239413, 'test/num_examples': 95000000, 'score': 7850.796292304993, 'total_duration': 26420.616366624832, 'accumulated_submission_time': 7850.796292304993, 'accumulated_eval_time': 18553.23108625412, 'accumulated_logging_time': 0.6840434074401855} +I0916 02:49:21.939463 139910841300736 logging_writer.py:48] [4597] accumulated_eval_time=18553.2, accumulated_logging_time=0.684043, accumulated_submission_time=7850.8, global_step=4597, preemption_count=0, score=7850.8, test/loss=0.126819, test/num_examples=95000000, total_duration=26420.6, train/loss=0.124182, validation/loss=0.124586, validation/num_examples=83274637 +I0916 02:55:19.055502 139954106807488 spec.py:333] Evaluating on the training split. +I0916 02:56:17.990623 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 02:59:15.373804 139954106807488 spec.py:363] Evaluating on the test split. +I0916 03:03:22.624985 139954106807488 submission_runner.py:516] Time since start: 27261.33s, Step: 4811, {'train/loss': 0.12213991612607121, 'validation/loss': 0.12426661725722801, 'validation/num_examples': 83274637, 'test/loss': 0.1264940008418033, 'test/num_examples': 95000000, 'score': 8207.194816350937, 'total_duration': 27261.325538396835, 'accumulated_submission_time': 8207.194816350937, 'accumulated_eval_time': 19036.80064892769, 'accumulated_logging_time': 0.7143452167510986} +I0916 03:03:22.648679 139910849693440 logging_writer.py:48] [4811] accumulated_eval_time=19036.8, accumulated_logging_time=0.714345, accumulated_submission_time=8207.19, global_step=4811, preemption_count=0, score=8207.19, test/loss=0.126494, test/num_examples=95000000, total_duration=27261.3, train/loss=0.12214, validation/loss=0.124267, validation/num_examples=83274637 +I0916 03:08:23.567865 139910841300736 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.0129254, loss=0.122698 +I0916 03:08:23.571117 139954106807488 submission.py:307] 5000) loss = 0.123, grad_norm = 0.013 +I0916 03:09:20.180603 139954106807488 spec.py:333] Evaluating on the training split. +I0916 03:10:19.124136 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 03:13:17.110412 139954106807488 spec.py:363] Evaluating on the test split. +I0916 03:17:24.263025 139954106807488 submission_runner.py:516] Time since start: 28102.96s, Step: 5029, {'train/loss': 0.1234449331014513, 'validation/loss': 0.12426789824103962, 'validation/num_examples': 83274637, 'test/loss': 0.12658910185611122, 'test/num_examples': 95000000, 'score': 8564.014400959015, 'total_duration': 28102.96358895302, 'accumulated_submission_time': 8564.014400959015, 'accumulated_eval_time': 19520.883132219315, 'accumulated_logging_time': 0.7445018291473389} +I0916 03:17:24.286655 139910849693440 logging_writer.py:48] [5029] accumulated_eval_time=19520.9, accumulated_logging_time=0.744502, accumulated_submission_time=8564.01, global_step=5029, preemption_count=0, score=8564.01, test/loss=0.126589, test/num_examples=95000000, total_duration=28103, train/loss=0.123445, validation/loss=0.124268, validation/num_examples=83274637 +I0916 03:23:21.182968 139954106807488 spec.py:333] Evaluating on the training split. +I0916 03:24:20.057318 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 03:27:17.605915 139954106807488 spec.py:363] Evaluating on the test split. +I0916 03:31:25.641960 139954106807488 submission_runner.py:516] Time since start: 28944.34s, Step: 5230, {'train/loss': 0.1241965349891888, 'validation/loss': 0.1242795916885101, 'validation/num_examples': 83274637, 'test/loss': 0.12659301345568205, 'test/num_examples': 95000000, 'score': 8920.203042268753, 'total_duration': 28944.3425180912, 'accumulated_submission_time': 8920.203042268753, 'accumulated_eval_time': 20005.3422164917, 'accumulated_logging_time': 0.7747159004211426} +I0916 03:31:25.665283 139910841300736 logging_writer.py:48] [5230] accumulated_eval_time=20005.3, accumulated_logging_time=0.774716, accumulated_submission_time=8920.2, global_step=5230, preemption_count=0, score=8920.2, test/loss=0.126593, test/num_examples=95000000, total_duration=28944.3, train/loss=0.124197, validation/loss=0.12428, validation/num_examples=83274637 +I0916 03:37:23.049354 139954106807488 spec.py:333] Evaluating on the training split. +I0916 03:38:21.946346 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 03:41:19.720523 139954106807488 spec.py:363] Evaluating on the test split. +I0916 03:45:26.460374 139954106807488 submission_runner.py:516] Time since start: 29785.16s, Step: 5455, {'train/loss': 0.12282474210147587, 'validation/loss': 0.12446043140474752, 'validation/num_examples': 83274637, 'test/loss': 0.12676063571215176, 'test/num_examples': 95000000, 'score': 9276.86423420906, 'total_duration': 29785.160937786102, 'accumulated_submission_time': 9276.86423420906, 'accumulated_eval_time': 20488.753309249878, 'accumulated_logging_time': 0.8045017719268799} +I0916 03:45:26.484064 139910849693440 logging_writer.py:48] [5455] accumulated_eval_time=20488.8, accumulated_logging_time=0.804502, accumulated_submission_time=9276.86, global_step=5455, preemption_count=0, score=9276.86, test/loss=0.126761, test/num_examples=95000000, total_duration=29785.2, train/loss=0.122825, validation/loss=0.12446, validation/num_examples=83274637 +I0916 03:46:04.392719 139910841300736 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.0203687, loss=0.127595 +I0916 03:46:04.395786 139954106807488 submission.py:307] 5500) loss = 0.128, grad_norm = 0.020 +I0916 03:51:23.000752 139954106807488 spec.py:333] Evaluating on the training split. +I0916 03:52:21.731760 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 03:55:19.530993 139954106807488 spec.py:363] Evaluating on the test split. +I0916 03:59:26.158385 139954106807488 submission_runner.py:516] Time since start: 30624.86s, Step: 5688, {'train/loss': 0.12061818726668612, 'validation/loss': 0.12426896104309743, 'validation/num_examples': 83274637, 'test/loss': 0.12654999859924315, 'test/num_examples': 95000000, 'score': 9632.654585123062, 'total_duration': 30624.85894203186, 'accumulated_submission_time': 9632.654585123062, 'accumulated_eval_time': 20971.911014795303, 'accumulated_logging_time': 0.8349716663360596} +I0916 03:59:26.180261 139910849693440 logging_writer.py:48] [5688] accumulated_eval_time=20971.9, accumulated_logging_time=0.834972, accumulated_submission_time=9632.65, global_step=5688, preemption_count=0, score=9632.65, test/loss=0.12655, test/num_examples=95000000, total_duration=30624.9, train/loss=0.120618, validation/loss=0.124269, validation/num_examples=83274637 +I0916 04:05:23.834312 139954106807488 spec.py:333] Evaluating on the training split. +I0916 04:06:22.797161 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 04:09:20.332171 139954106807488 spec.py:363] Evaluating on the test split. +I0916 04:13:26.113044 139954106807488 submission_runner.py:516] Time since start: 31464.81s, Step: 5911, {'train/loss': 0.12228138900259167, 'validation/loss': 0.12427444083574461, 'validation/num_examples': 83274637, 'test/loss': 0.1266273077835886, 'test/num_examples': 95000000, 'score': 9989.594222784042, 'total_duration': 31464.81358885765, 'accumulated_submission_time': 9989.594222784042, 'accumulated_eval_time': 21454.18979382515, 'accumulated_logging_time': 0.8636422157287598} +I0916 04:13:26.136420 139910841300736 logging_writer.py:48] [5911] accumulated_eval_time=21454.2, accumulated_logging_time=0.863642, accumulated_submission_time=9989.59, global_step=5911, preemption_count=0, score=9989.59, test/loss=0.126627, test/num_examples=95000000, total_duration=31464.8, train/loss=0.122281, validation/loss=0.124274, validation/num_examples=83274637 +I0916 04:15:10.173542 139910849693440 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.00579653, loss=0.124401 +I0916 04:15:10.176532 139954106807488 submission.py:307] 6000) loss = 0.124, grad_norm = 0.006 +I0916 04:19:22.939686 139954106807488 spec.py:333] Evaluating on the training split. +I0916 04:20:21.638686 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 04:23:19.076370 139954106807488 spec.py:363] Evaluating on the test split. +I0916 04:27:25.910341 139954106807488 submission_runner.py:516] Time since start: 32304.61s, Step: 6181, {'train/loss': 0.1236555678991646, 'validation/loss': 0.12430735434169013, 'validation/num_examples': 83274637, 'test/loss': 0.1266926717426501, 'test/num_examples': 95000000, 'score': 10345.668259859085, 'total_duration': 32304.610867500305, 'accumulated_submission_time': 10345.668259859085, 'accumulated_eval_time': 21937.16048359871, 'accumulated_logging_time': 0.8935234546661377} +I0916 04:27:25.935933 139910841300736 logging_writer.py:48] [6181] accumulated_eval_time=21937.2, accumulated_logging_time=0.893523, accumulated_submission_time=10345.7, global_step=6181, preemption_count=0, score=10345.7, test/loss=0.126693, test/num_examples=95000000, total_duration=32304.6, train/loss=0.123656, validation/loss=0.124307, validation/num_examples=83274637 +I0916 04:33:23.228075 139954106807488 spec.py:333] Evaluating on the training split. +I0916 04:34:22.206636 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 04:37:19.584939 139954106807488 spec.py:363] Evaluating on the test split. +I0916 04:41:28.682484 139954106807488 submission_runner.py:516] Time since start: 33147.38s, Step: 6392, {'train/loss': 0.12257261110970631, 'validation/loss': 0.12462830525078661, 'validation/num_examples': 83274637, 'test/loss': 0.12701918875234502, 'test/num_examples': 95000000, 'score': 10702.254277944565, 'total_duration': 33147.38299417496, 'accumulated_submission_time': 10702.254277944565, 'accumulated_eval_time': 22422.614903450012, 'accumulated_logging_time': 0.9256329536437988} +I0916 04:41:28.707011 139910849693440 logging_writer.py:48] [6392] accumulated_eval_time=22422.6, accumulated_logging_time=0.925633, accumulated_submission_time=10702.3, global_step=6392, preemption_count=0, score=10702.3, test/loss=0.127019, test/num_examples=95000000, total_duration=33147.4, train/loss=0.122573, validation/loss=0.124628, validation/num_examples=83274637 +I0916 04:44:09.167636 139910841300736 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.00558099, loss=0.116978 +I0916 04:44:09.170842 139954106807488 submission.py:307] 6500) loss = 0.117, grad_norm = 0.006 +I0916 04:47:26.503743 139954106807488 spec.py:333] Evaluating on the training split. +I0916 04:48:25.436096 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 04:51:23.267867 139954106807488 spec.py:363] Evaluating on the test split. +I0916 04:55:31.106278 139954106807488 submission_runner.py:516] Time since start: 33989.81s, Step: 6612, {'train/loss': 0.12285226374716535, 'validation/loss': 0.12421706379676087, 'validation/num_examples': 83274637, 'test/loss': 0.12660717286489387, 'test/num_examples': 95000000, 'score': 11059.34743642807, 'total_duration': 33989.80683994293, 'accumulated_submission_time': 11059.34743642807, 'accumulated_eval_time': 22907.217544555664, 'accumulated_logging_time': 0.9574995040893555} +I0916 04:55:31.131070 139910849693440 logging_writer.py:48] [6612] accumulated_eval_time=22907.2, accumulated_logging_time=0.9575, accumulated_submission_time=11059.3, global_step=6612, preemption_count=0, score=11059.3, test/loss=0.126607, test/num_examples=95000000, total_duration=33989.8, train/loss=0.122852, validation/loss=0.124217, validation/num_examples=83274637 +I0916 05:01:28.392152 139954106807488 spec.py:333] Evaluating on the training split. +I0916 05:02:27.401229 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 05:05:24.939988 139954106807488 spec.py:363] Evaluating on the test split. +I0916 05:09:32.544740 139954106807488 submission_runner.py:516] Time since start: 34831.25s, Step: 6851, {'train/loss': 0.12415708162197589, 'validation/loss': 0.12422422161349748, 'validation/num_examples': 83274637, 'test/loss': 0.1264844068511963, 'test/num_examples': 95000000, 'score': 11415.9016726017, 'total_duration': 34831.24528861046, 'accumulated_submission_time': 11415.9016726017, 'accumulated_eval_time': 23391.370171785355, 'accumulated_logging_time': 0.9887738227844238} +I0916 05:09:32.570024 139910841300736 logging_writer.py:48] [6851] accumulated_eval_time=23391.4, accumulated_logging_time=0.988774, accumulated_submission_time=11415.9, global_step=6851, preemption_count=0, score=11415.9, test/loss=0.126484, test/num_examples=95000000, total_duration=34831.2, train/loss=0.124157, validation/loss=0.124224, validation/num_examples=83274637 +I0916 05:13:32.127719 139910849693440 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.0259948, loss=0.120191 +I0916 05:13:32.130979 139954106807488 submission.py:307] 7000) loss = 0.120, grad_norm = 0.026 +I0916 05:15:29.106900 139954106807488 spec.py:333] Evaluating on the training split. +I0916 05:16:28.041907 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 05:19:25.202672 139954106807488 spec.py:363] Evaluating on the test split. +I0916 05:23:33.667465 139954106807488 submission_runner.py:516] Time since start: 35672.37s, Step: 7067, {'train/loss': 0.12408029630112555, 'validation/loss': 0.12420217021317523, 'validation/num_examples': 83274637, 'test/loss': 0.126472298812545, 'test/num_examples': 95000000, 'score': 11771.739629983902, 'total_duration': 35672.36800336838, 'accumulated_submission_time': 11771.739629983902, 'accumulated_eval_time': 23875.930784225464, 'accumulated_logging_time': 1.0206894874572754} +I0916 05:23:33.690746 139910841300736 logging_writer.py:48] [7067] accumulated_eval_time=23875.9, accumulated_logging_time=1.02069, accumulated_submission_time=11771.7, global_step=7067, preemption_count=0, score=11771.7, test/loss=0.126472, test/num_examples=95000000, total_duration=35672.4, train/loss=0.12408, validation/loss=0.124202, validation/num_examples=83274637 +I0916 05:29:31.782987 139954106807488 spec.py:333] Evaluating on the training split. +I0916 05:30:30.691082 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 05:33:28.528238 139954106807488 spec.py:363] Evaluating on the test split. +I0916 05:37:36.157187 139954106807488 submission_runner.py:516] Time since start: 36514.86s, Step: 7298, {'train/loss': 0.12081371002780912, 'validation/loss': 0.124043713182727, 'validation/num_examples': 83274637, 'test/loss': 0.12635201827826248, 'test/num_examples': 95000000, 'score': 12129.126768112183, 'total_duration': 36514.85768580437, 'accumulated_submission_time': 12129.126768112183, 'accumulated_eval_time': 24360.304978609085, 'accumulated_logging_time': 1.0505201816558838} +I0916 05:37:36.182031 139910849693440 logging_writer.py:48] [7298] accumulated_eval_time=24360.3, accumulated_logging_time=1.05052, accumulated_submission_time=12129.1, global_step=7298, preemption_count=0, score=12129.1, test/loss=0.126352, test/num_examples=95000000, total_duration=36514.9, train/loss=0.120814, validation/loss=0.124044, validation/num_examples=83274637 +I0916 05:42:50.770550 139910841300736 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.00540812, loss=0.120243 +I0916 05:42:50.774267 139954106807488 submission.py:307] 7500) loss = 0.120, grad_norm = 0.005 +I0916 05:43:33.421471 139954106807488 spec.py:333] Evaluating on the training split. +I0916 05:44:32.337079 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 05:47:29.957145 139954106807488 spec.py:363] Evaluating on the test split. +I0916 05:51:37.193573 139954106807488 submission_runner.py:516] Time since start: 37355.89s, Step: 7525, {'train/loss': 0.12114051760387055, 'validation/loss': 0.12385989917627897, 'validation/num_examples': 83274637, 'test/loss': 0.12615459975569876, 'test/num_examples': 95000000, 'score': 12485.66578912735, 'total_duration': 37355.89405155182, 'accumulated_submission_time': 12485.66578912735, 'accumulated_eval_time': 24844.077082633972, 'accumulated_logging_time': 1.0818431377410889} +I0916 05:51:37.216840 139910849693440 logging_writer.py:48] [7525] accumulated_eval_time=24844.1, accumulated_logging_time=1.08184, accumulated_submission_time=12485.7, global_step=7525, preemption_count=0, score=12485.7, test/loss=0.126155, test/num_examples=95000000, total_duration=37355.9, train/loss=0.121141, validation/loss=0.12386, validation/num_examples=83274637 +I0916 05:57:35.468314 139954106807488 spec.py:333] Evaluating on the training split. +I0916 05:58:34.213041 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 06:01:31.849761 139954106807488 spec.py:363] Evaluating on the test split. +I0916 06:05:40.004461 139954106807488 submission_runner.py:516] Time since start: 38198.70s, Step: 7736, {'train/loss': 0.12135035106859121, 'validation/loss': 0.1238473428037413, 'validation/num_examples': 83274637, 'test/loss': 0.12608699735605342, 'test/num_examples': 95000000, 'score': 12843.221409320831, 'total_duration': 38198.704989910126, 'accumulated_submission_time': 12843.221409320831, 'accumulated_eval_time': 25328.61328649521, 'accumulated_logging_time': 1.111832618713379} +I0916 06:05:40.029068 139910841300736 logging_writer.py:48] [7736] accumulated_eval_time=25328.6, accumulated_logging_time=1.11183, accumulated_submission_time=12843.2, global_step=7736, preemption_count=0, score=12843.2, test/loss=0.126087, test/num_examples=95000000, total_duration=38198.7, train/loss=0.12135, validation/loss=0.123847, validation/num_examples=83274637 +I0916 06:11:37.010999 139954106807488 spec.py:333] Evaluating on the training split. +I0916 06:12:35.887506 139954106807488 spec.py:346] Evaluating on the validation split. +I0916 06:15:33.616516 139954106807488 spec.py:363] Evaluating on the test split. +I0916 06:19:40.468913 139954106807488 submission_runner.py:516] Time since start: 39039.17s, Step: 7941, {'train/loss': 0.12220586679590756, 'validation/loss': 0.12397896913144073, 'validation/num_examples': 83274637, 'test/loss': 0.12629493782991108, 'test/num_examples': 95000000, 'score': 13199.509422063828, 'total_duration': 39039.16946601868, 'accumulated_submission_time': 13199.509422063828, 'accumulated_eval_time': 25812.071299791336, 'accumulated_logging_time': 1.142986536026001} +I0916 06:19:40.495650 139910849693440 logging_writer.py:48] [7941] accumulated_eval_time=25812.1, accumulated_logging_time=1.14299, accumulated_submission_time=13199.5, global_step=7941, preemption_count=0, score=13199.5, test/loss=0.126295, test/num_examples=95000000, total_duration=39039.2, train/loss=0.122206, validation/loss=0.123979, validation/num_examples=83274637 +I0916 06:20:53.947676 139910841300736 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.00660578, loss=0.120311 +I0916 06:20:53.950664 139954106807488 submission.py:307] 8000) loss = 0.120, grad_norm = 0.007 +I0916 06:25:36.618233 139910849693440 logging_writer.py:48] [8183] global_step=8183, preemption_count=0, score=13555.2 +I0916 06:25:51.488634 139954106807488 submission_runner.py:857] Final criteo1tb score: 13555.211057901382 diff --git a/logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..e1d9f67d --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,39 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/loss,test/num_examples,total_duration,train/loss,validation/loss,validation/num_examples +1860.652927160263,0.0,9.658968210220335,1,0,9.658968210220335,1.3495619400801808,95000000,1870.628292798996,1.3445777583756644,1.3570842317910403,83274637 +2913.9182076454163,0.0563101768493652,366.55974078178406,200,0,366.55974078178406,0.1314415748999344,95000000,3281.604463815689,0.1268694460864347,0.1289985184971938,83274637 +3963.20534324646,0.0872905254364013,722.8921821117401,402,0,722.8921821117401,0.1304406822643079,95000000,4687.963960409164,0.1292311273234526,0.1279783523133029,83274637 +5008.411679267883,0.1173408031463623,1079.2291326522827,601,0,1079.2291326522827,0.1291329746458354,95000000,6090.243111848831,0.128460360749033,0.1267814876495373,83274637 +6073.168746471405,0.1463634967803955,1435.5280559062958,802,0,1435.5280559062958,0.1287239919322766,95000000,7512.024815559387,0.1261564336058258,0.1262946168819717,83274637 +7101.967989206314,0.1746838092803955,1792.1379549503326,1008,0,1792.1379549503326,0.1285700634242007,95000000,8898.164870262146,0.1273649113555809,0.126224402964769,83274637 +8123.249813318253,0.2063307762145996,2148.3401172161102,1220,0,2148.3401172161102,0.1281569895374499,95000000,10276.384850740433,0.1249698756503415,0.1257965216111919,83274637 +9134.081369876862,0.235431432723999,2504.3137986660004,1434,0,2504.3137986660004,0.1278985934318141,95000000,11643.921679496763,0.1262998824987502,0.125697069969302,83274637 +10115.627338647842,0.2652029991149902,2860.8038444519043,1636,0,2860.8038444519043,0.1279189863202546,95000000,12982.689307451248,0.1236481840879651,0.1255930255676328,83274637 +11054.704303264618,0.2939579486846924,3217.747561454773,1846,0,3217.747561454773,0.1279417368135151,95000000,14279.446210861206,0.1225691066353856,0.1255440421935849,83274637 +11959.5324447155,0.3234055042266845,3573.622041463852,2059,0,3573.622041463852,0.1278243510589599,95000000,15540.881484270096,0.1235477273606574,0.1253004452267213,83274637 +12773.260489463806,0.3537471294403076,3929.251423597336,2264,0,3929.251423597336,0.1276238236686305,95000000,16710.97428059578,0.1214590713949025,0.1251349367273924,83274637 +13513.194631576538,0.3836984634399414,4285.6617822647095,2470,0,4285.6617822647095,0.127745887270315,95000000,17808.054710149765,0.1235114090716656,0.1252825022753245,83274637 +14171.295335292816,0.4115707874298095,4642.696635723114,2695,0,4642.696635723114,0.1273950589214124,95000000,18823.92814230919,0.1228290405714404,0.1249627153190607,83274637 +14685.586983680723,0.4423439502716064,4998.295198917389,2915,0,4998.295198917389,0.1272692146180805,95000000,19694.56010222435,0.1245766903279074,0.1247602831126512,83274637 +15168.773005723951,0.4729907512664795,5354.927243947983,3144,0,5354.927243947983,0.1272691373829088,95000000,20535.125536441803,0.1250313014233972,0.1247420626848063,83274637 +15651.562380552292,0.5058600902557373,5710.522954463959,3366,0,5710.522954463959,0.127170024838578,95000000,21374.258639097214,0.1235264876040958,0.1248940205799324,83274637 +16135.808561325071,0.5355989933013916,6066.370519399643,3588,0,6066.370519399643,0.127062874858816,95000000,22215.099447250366,0.1222109906846291,0.1247722997757567,83274637 +16618.414626598358,0.5651638507843018,6422.8106808662415,3792,0,6422.8106808662415,0.126931378600753,95000000,23054.88243794441,0.1243355281566004,0.1245174247297344,83274637 +17102.797516345978,0.5944151878356934,6780.02803850174,3988,0,6780.02803850174,0.1269691514418752,95000000,23897.213665246964,0.1214501012929081,0.1245933773395324,83274637 +17585.180767774582,0.6244139671325684,7135.8886506557465,4189,0,7135.8886506557465,0.1267508165151495,95000000,24736.193083524704,0.122534525138587,0.1244867060856099,83274637 +18067.606957674023,0.6540706157684326,7493.290529727936,4386,0,7493.290529727936,0.1267406875634444,95000000,25576.752193450928,0.1228863687585938,0.1245126588331126,83274637 +18553.23108625412,0.6840434074401855,7850.796292304993,4597,0,7850.796292304993,0.1268185528223941,95000000,26420.616366624832,0.1241817561561066,0.1245861310388407,83274637 +19036.80064892769,0.7143452167510986,8207.194816350937,4811,0,8207.194816350937,0.1264940008418033,95000000,27261.325538396835,0.1221399161260712,0.124266617257228,83274637 +19520.883132219315,0.7445018291473389,8564.014400959015,5029,0,8564.014400959015,0.1265891018561112,95000000,28102.96358895302,0.1234449331014513,0.1242678982410396,83274637 +20005.3422164917,0.7747159004211426,8920.203042268753,5230,0,8920.203042268753,0.126593013455682,95000000,28944.3425180912,0.1241965349891888,0.1242795916885101,83274637 +20488.75330924988,0.8045017719268799,9276.86423420906,5455,0,9276.86423420906,0.1267606357121517,95000000,29785.160937786102,0.1228247421014758,0.1244604314047475,83274637 +20971.911014795303,0.8349716663360596,9632.654585123062,5688,0,9632.654585123062,0.1265499985992431,95000000,30624.85894203186,0.1206181872666861,0.1242689610430974,83274637 +21454.18979382515,0.8636422157287598,9989.594222784042,5911,0,9989.594222784042,0.1266273077835886,95000000,31464.81358885765,0.1222813890025916,0.1242744408357446,83274637 +21937.16048359871,0.8935234546661377,10345.668259859083,6181,0,10345.668259859083,0.1266926717426501,95000000,32304.610867500305,0.1236555678991646,0.1243073543416901,83274637 +22422.61490345001,0.9256329536437988,10702.254277944565,6392,0,10702.254277944565,0.127019188752345,95000000,33147.38299417496,0.1225726111097063,0.1246283052507866,83274637 +22907.217544555664,0.9574995040893556,11059.34743642807,6612,0,11059.34743642807,0.1266071728648938,95000000,33989.80683994293,0.1228522637471653,0.1242170637967608,83274637 +23391.370171785355,0.9887738227844238,11415.9016726017,6851,0,11415.9016726017,0.1264844068511963,95000000,34831.24528861046,0.1241570816219758,0.1242242216134974,83274637 +23875.930784225464,1.0206894874572754,11771.739629983902,7067,0,11771.739629983902,0.126472298812545,95000000,35672.36800336838,0.1240802963011255,0.1242021702131752,83274637 +24360.304978609085,1.0505201816558838,12129.126768112184,7298,0,12129.126768112184,0.1263520182782624,95000000,36514.85768580437,0.1208137100278091,0.124043713182727,83274637 +24844.07708263397,1.0818431377410889,12485.66578912735,7525,0,12485.66578912735,0.1261545997556987,95000000,37355.89405155182,0.1211405176038705,0.1238598991762789,83274637 +25328.61328649521,1.111832618713379,12843.221409320831,7736,0,12843.221409320831,0.1260869973560534,95000000,38198.704989910126,0.1213503510685912,0.1238473428037413,83274637 +25812.071299791336,1.142986536026001,13199.509422063828,7941,0,13199.509422063828,0.12629493782991108,95000000,39039.16946601868,0.12220586679590756,0.12397896913144073,83274637 diff --git a/logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/trial_1/events.out.tfevents.1789500541.fe7c58f35b35.38.0.v2 b/logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/trial_1/events.out.tfevents.1789500541.fe7c58f35b35.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..f9b00047bda9378050e6f6e261acdc8cfc5174e5 GIT binary patch literal 57683 zcmbWAcU+DA|Hnfbr)?xeWsiiG7KJFA$}W{;q@^99qBO`Vva>1`xs4Vjg^G|p?hr!u zDv{Ce+d1y*^Lt(6bWY!Y@9+Ed_&l%A^?ZHKIh}JEAtCtRgPRtwWF;glG!~^j9N9%- zzK`<~Pn|IVuAY7-x;DLdJ^WlfeZ71Zx_brcxOn-v>iGLDao2He`tQDufj&$8TzyuI z9P&>4Z(8lK@-eas4QI8Q(gtYpGNp4;McoDzTV#e`4r6= zva1LG4d}<~!;@R&?ls@p-O<<2)%(AQrvEcUG&&wTq5n5xo?a(=*_fFUUl;#x{mQKd z^yT&D@%);u$>9Ibq1lzj5&BwwyYb(QSm|M(WyiTt8QQ!7Jf#3<_azIQ{g!xP(5F*J z@q#>6l@A~JH%Q`0QM#<22Nk5x)8%#ge?gx99*(X-&K}hR!;SU$+ zpg$GJzs7#9zJ37bQ$T||GduE?>EAVV>6u|K+i5iwp~KVWsqiD%R+FQ98dr04zI@r> zzfm3z-KNU^j-;Y`@HBaCeVcBw|F)_}Y?qM$)8JAz!sze#!QhoMWy|8I@qxVlJVie* zzyEFb0)HR=rT#Cc>+RQ;kP~ZoHD}DwP?{2&qa{fGz-)>liKGS{7t-j(cz_RTqc#s zZA&QLpdt2)kD2@pX_xTknQTHf6*7o7kf-eJFjO8dP8OQ1T zO}cnxW`peIBNP7fj?oE;xZMYe z!gq576s?*4S3M>juV%Z9Yx96YzmA#~kz#7ZhetqR_nV5~rjW3JTVqQUZzE95o$Dr` zXv3tC;2lAV65T}`fTF>Lnr5f?P4Dqm-`?f|#jT}O1UH2RQbdi2?Fbb2qJsq#l1vIc z`(&h;>gtvO6t-KaX?BW6I>p7g_Z9)ghP6f1QSk-{2E+0zN}h6A;+YMjPOBLvpO)&&1O#1jf z0*b{6Uj-EMOp1Wrj!3cm%Tr&VDBeR&i%9YHZkY~HbV;TnxG5}GTf0Y&m{DuSCL6e(_p*J%MoL>~(Qg$k3RZSf!cVBub$9n*l~x(+ohB88FG ztJ6Sn&5(-Vriem{?Ft=qfTCgaQ~^ahCdHVel}KTvJ@gAuESyM9i%1c^>6#2soVTMQ zxG6RvMaQwbJ_5z)nT`UA_DqT%k@-j=IbltApm2Ajrr9YNuRsnT>b(#s<}IQkxG55m zVtFR-8BqARdkQEzFeyHn9zqK16)xz7lRq`hPSHSr9kn9$?Jb}fyON6Frbt2x?Sd8c zKyfO4rGTO%lj6_2%}DX8&6yXl7aLAd)9e)A=@e-nt=0p@k+W0;H$^H^49)%;0~85) z>jf05Op4vp!jR&7Ubz=gm=;sh>=bo$insH!rvOFoD^vtG#c`yV^!?C!pqTVvmw=)Z zlj2i`4^nL7Iobe4?Ne%+o#Hi}B34oM4N%m-rXsj0a*!fo+Yo$zaj79qK%vH@2$ETd z6y4X@P6CSFlIj0Vvr|ONm10E0?}_?Ap(Ia5a8nc_MdhTV{;(G{I(dS1bY@ccNIT&Y zf*-ya2^6D^sA+Zz^Qq4;V!=lLX+W{ff{Ng#C`Af`Z0SotaoVm-K+%Os@ycldE@Acr zzaBtQ>Of7iQ@o_#QE}4U!hnLekc!}@s6dJ?eNL_biawtA1r%MG6dUd?LJCivL6$)A zEP$G3r+7!FIN=aMw6n?!7 zZGgfwg^J*&_=FT`F^5+J#o6Pv0tyW#h0VqZNU=Js*KMHqoJ&o!Qz-T?z=-L8DieU> z$$2V*o8miCnC(0m02H(9gTL|jLX%1H^{6E-q1)TcUO=HMvHZVjc8bcfl^F5wJ#7P^ z@ROk;xG5w?!Dr@6v%Y)-iK~Oo~rFGz#Boo*jXrJd2uUr}*$C9x39JB?Ez? zx`2w{rjSI6J69&$1&V!T@dAn-ObXMHBXJ1^bJn5V(zmE-c8XTDJ24`->f1h`82m34 z!A&896v?fn6oDdPkkWVlUi4&AOpG(eB{Y5=s|aU&o#eKyg@gzz_aj^k!0c)a&9BK1lxD2Na_J30Yx7s#o1&XTtb54@;IQ_twv3=Q^*alphaZP7zz|xT2ur#g&I;E zPSL>k7kxYRZs6}lUna%n`hK_s3*%nnfkIc4nr5ev2-U)fAnVM#&P@Pdf91g4>1+usoOK(yU+!Vc#VrbbV z{5+~c@vwkm0F%P*o+eU^sFLgh6rP=^X?6<#(RY!eV*QU%uoqV~s0eO~en_#cnflLawcHNPppmijk7jv&s)9e(}Ez6MN$JwXlK%sw+ir}WuMhf4(=kXPY zlixl8g*KBS|B)I}sP4bj9`@qfN@|*&qWiU6q&S!3mkku{BB=;&iXli5b|VA72NLvA z>o@sQe8nI)rU{2NX*rdi=KlZVG**Sh&nH4DP62 zVL5^j9VUfwHEk~%bEMOOLNA7zW~W%W^&?VLJ}!L)6j!%U5!@6;Nb#r375oh3bd2sF z{$31bQq0%tgnN=bTkJCH)DcxN?G80?}VxGBt#!m9Vb^IJ%yTPr`UcZ5h;#zNV^FX{xhivZVF4Jct2^LIZzDNJ0qabWm42$ zYL9!NP$_>0C~VEBX?BWHn{Ob+g4%a|fFi<@ir}WOMv9G3OYk+VmB~nnB~9NE)nigT zSfzp#{<{au0!7FeYMPzm;Daora7zEZ11K&{q9V8{CL)E!;yOI_<~rvHDD;^WJF^s! zB1=VeC{TQIqo&y@a=v&Y#pev!SfH@-r6RZ~rXs~NrRP)OjyfMVr4@fK444$9jWmjR z#d2$b;zA-d%}!zYn0EZf?Hh&PxGPAaBDg8+kRsL8B^oFe>fI4g7&0lwuAouuZR36q zDBhS+)9e(TA}Vn&r1vM>fW5eHNkwo|%tDH!V{P#LMOLn3YyMsgV^Sn9kjFc!Ds_A$ zP$U*n)9e)4-7Jt|{l>-k{itV`sR(Wgd!%@K;@eT!i})XJ1r$b13i&H?ND=L3k_i-% zt>67O%}$YH*aah;W|Z(?FB0Ua2yO}|q$r6BJPs5Nom|@R_rjP-5iUjB3&lE#{Xj86 zlbU9yXuLWcDR?J3;$Mm@`cV#_v7Q(>xQPh-xR%mQP{Eq^QlHy=krw_`V7#9;#E*>=Y_{ z6>%>lKPV0XiiX})1UH2@Qs~6Q;$Q7gCax1u3};fvgwgIORi4`(pvc=#O|w(92|I!m zcCjWCfg(1Iir}X3LkeC8i;=Jw&WGDc@%Lf`lVXo6Z7)_UOF98Xw`0^aI|bwK7hAHF z{sD@~SyTi!MG#Uft;x6!6w!V63n6;R}z zp(3~`!jVF1sxp3hFk`2s41X_1G56wNI_-|qtN!v6C=?G;)9e%zGHZ}R-^BbBP%KHO zBDg8mB1P;=y9(HgFTD!{6r-6GYNKg;QBgbV22e~IOii;>x8~9rom0?r_ zH$@~;4El9zCs1VX7%j`+3ri-2!Y|rhOp;&G3MdK=P}A%bQOB!sFBaP5DZpNsrc)8z z6j4YK^3td^Q0S$U3n<1gDU$n3;Sy92KZ*c~!c1zKog&BP3Pz-ibv6Tv*V$AAH$@Cm z{8N7P1yCHgHI2vLi?K`!O-~xdn1W;f0)=%YHO)>jd-@N21v1!bC!Tt%s;LNWiuFj* ze?W^JUV*(n&Oo{hvvW7vx`+o=d{ia4a$?rPr! z_9CaJqa1%PteF(YmeVK>W*^7TK(1?3)9e(zvhwhb^6<#Uzp)N9pdz>_HX%jDm+t$4 zVs(dC0*Y}=iqmGaJIcZ4(mdFUoNm-KJB7i}&v-`-bR3MIHdpkfBDg8GAjQ1Vrk+3% z+0{XwzZc_~6pG7fP3u!${R$|8dr{Nu6s3){Z+U$=v-la@QBm4d1UE%IQdB(I5dsu; z8C3#`2}}x;Hni8YZ*pbyfx`PVHO)@pmtKT-l;paUjX+V5PepK3Y(t93f|9X7VHo74 zz~76BOp0}jXz!A0+K+w$6qTXWG&@DSRsh~nd$e7a0>!{+DuSCL5h*6-NYntuoUyGG z`4p3w6hq@_pX4|_4h{o~E>oy!c8VjuUvV$iJluw_Kq6;S5!@6zk;3Ch+Do7aZo6DS zF_}prb5aSHFzU->{64L(A~nrUaYOGLM%*8`GadHgvMLq9O|cs(-kbWY0*c{^QcC>2 zn8KupGpA8()J?;`cRyF9rr9a>PVA3&)aw_{V}K&3I~BoAu@@?JB1eSHSWd09h2Vzh3hjaf}0`0^56b}b~K#IV#!!H9x@GB~Uo8kac$W)c%H&I7F+a#c{VN#S& zrcrFDz5W|03g1xE>=bLh^u{}C^9ajApvd|{MQ~FbLW&pCkAvWjI<{9yg})cJObYu; zw9f`5VvcPxG9p6VwlDv{G9gv*#iO!J0``JQ?#ZX zT(YnlD3)BLrr9Yf>#yS-Rp5KF1Sr~EqawH|Qjy|@%b)|W7YA;1Y{%b==}ZbGZQ5SE zUv*?RP;7cgO|w(jT0X%$s$G$Y998C7_tDMn!N_WFSTEzZ!b57ityV+w=EgCX-@eCG8c6{Wwp2 zP3!xBnr5d^QZB(eYJ-HybfB<)N=0x}97Bp(o3~{GMNwg_fMOPtB7Yc-qElGzHrNaE zGHRNgqV&>lyrYyC8le|1H>e11isMMJa8a=#P$)c6>cHQN*-VPfXK7w&-s}DiC{Dei zrr9Z~=y%ko-%A(4UX1@lMQ~G`K#KQo%J7$NDy6#w6mysqZQjY^?QrF@gf~!>R#4OI z6z9T@V?^(}rqV!hqh-!V75MxQqVKvm<{m<}xWx4WzxlXf<>P{$9_wo|t9xiY%nqpZcyd+);LMhXfS%Op43CG>QSkGta_agl(s$*(sjW z?`xXlx}sx z@0kxSq$0Q}3XwwkgbaQjb?!_LHU3^WGb!eow?T?;nhpJcLiZ9i%}x>ZJO}TnGlM>2 zN42S-BDg7vkm5MYtyy#22qsApx#KT@}+D}chQ%sKCffO%C^r-^7 z9ga{D+!U9PV*D3{GeB|9IYmIRfJyP~q$DmO=)nB1Kyk^9nr5e%@q+eUHTwi@FQ6#% zp(3~`N|B=HSbh95$+$IVY~7l^t5(YSKtts>tq`W;b)EU}n72L~NQkf}8V*sTU6!TA z_cy_F?9>uSQi~pGXsqid4-agV+f5SrSDNum!+nV~aek{O>6!g~_*`lBK^s}UgCs-D zhZ_Eg9jHCzZ&pHGg`=!Q8p&cl)iBvAdo=ttH*9ulbMFH*(6%^&e#)l#@tqajbD4{`a|VJ9y4z`7M$n z-m?u2+w8Ogk#G9goLt)E;FdOA4;39Soo3F(pAnD1!EYfXM3{p`^0dRQqiU;>;Glj4 zNotXUIS~#P;NYtDB$3_05nE`r|7GZpbZ{_dE6EVk!9zDcR)B-0J4qJP!7jP___R%< zxhnU;!6W<0ybuSoR&T+(+o3485*(~eAwi-#$XMROP_tj)VB;~8+Ps6~P6pyCpXhZM z1`bx7CV3(`=yZ|xo%gkaOZ$U^d-6#nw}WNq;E>z4-r%6hMUo<(gUi=8~QOObp{bU4=5SUqxHd29ea}9h_5^kE`6)czZWEsJWWtiR9qas*d;;Q1(kNS8%X8l0-yeJ6OZ=G~SGtw4Xnx&(!(@cX-HNGR}UW zS51qmDqFGAZ(Sa^0(`Og>3-mC4tot#Nx)+tkNH znK%WK5MgEV-GWvo!*?|rLYdrOMUq-nCK~Ss9D_?86+sf&%j8K4txP6Gw?6`9626{f zh*>5^ok}vGOm=Q1Sq*QIaQOnb02Vp^+5w0>rs+ibQgY6N38? zwQ27ZD3cv$NQ!u6k`}i#6C5<3;n8ndlY+)r%yuJGjl1=AiwYjgP^>$NnTkOa~t|j*aPeWeZ{Xm&LnK8!2W_Or zXn})O`BPT;Gaj?2L!JKE`W;*F;(I4D_4LWDVZHjCz<$twRz;9$yilGGvx-8EEy zfrBIOl0;n#dd`_~M4z`U@$D7i-*N6acF#Qdg z7vkXYj|S*qS#Iq&a8T(p2@=&o#_}G%v)ly^Ua2Rk%{yqDqll}#npNWl4lema@MRn)PK3n+?R$*frcPJChk0e9PG6~wX^A(iIl{%8eER#p&x>zO|SL+TynW+9E^Fqp` z=;3(O)Zgt>FqFxF)~%?W7qv_n%d_1pCl6&3Btuf0FOzV^?zqb9#wtlrCMOh0o=9bK zymTFw$=>IERY9DZjwF&>oN^S$;h#^>piEw=lN9mF#N_0=3^>qv7uDbI zF&+HYbsRc4sPdH_I9L`y=7l&|*3A*`?lBGydf?#P zl_W@12N}!z-lM}Qa8Q3ONp0T28+qEeO4rApZ^6Ob7?LNFgQ;QL(7~MR$(z8zJDW)) zr-Owr(Lue2lpt`hJdvb`=V0<69cehwXY1P9@ExpR`8PR*jt=^3WKVPZg2nZzn{mC1G~+JSyzB<@FcR-m`FCsyrmI-5d zbviPoP$tVgNowfrGWvNRX%wGM48O7=H#FoHdK2Ht(QZ zf)%dPva7{YaB$~5k|&abv$~%~2Tv{j`4b$x;z}Yp9Sl{#eK7LQvjGR!xsw#}95m>x zJP#by>M?6gP?Lkn?YSJ>@bwouXzhKc8eUX43?w1K98?@nbMW=Ri?QIKt3F9;k%RYM zwjTu!?lL8b><-?zPIEBiq5M~HFk%$R5Yxfx{4smL!PetQ7Sq8A?;~*UpPkZN1P<<= zLgs}ysID1{cekHq<0Wt~e>w>g)j`JcV)twx2@X!MC#lUlSklWDSDF3N=LR^~&xPcP zW%9wCga|7Wzjw4hpmp)?vDBty(HDStf67RqFhJ;`F0$<~gW@g^?Px%mppB*=x# z3n`O-mA0a$xl)_0piHuskRVaZgt5FkQ&e6-nWTD?)aJ`X$HM_vIr!LqM<|n+K$0g? znJmtw{n*IyL6oz?awIQ0|$MclGNrMoE_kbtCU(is2Uu!ts!|LIass&86N2OWx8p?;T8Ik zL~=Sfg!apkOF2t_fP-~)Bt<+2H-$fV0tfo9xc5PP2Xk2d<;AFPG-v*mJP-kQc*Axw z&VHajX#9x>y3L*e_+7HTNhE{wnuMBSFI9g?D}gd;^-F!wil#CtRpTlX0~y+D`Zgam ze?ghtY1@qo5mqJ_f@l}!(1W)XP$p3dB&kJZ@}zZvIb7<29Y`X3ncSn7Nupe1C6r0` zt|UXuGKso9O##XzPK#tQ%Vg=y6x^^rhbJ^bne^&M=7p4rg=Z3Ksyt`cbSRS{IwVNc zGGQ!Fes;Y5;JD_>a6B^(c!32S;rqk(>@X+2B5O_?)mD9Q?DJq=@HW z>x*VD;Xv0h8>|!3bPi^){FAZr#;@`7v6C;`AB8*o^B6MDexOg-REh_B>wzElz%^+% znPhNYlfpT8O|A{wzXr-grEAw4e2}&3TxC);t^slEI~T2lGEwhMLWGscz9?Fm#1w2P zg);HdCP^(Slj5p5-{Ddh>5)YCGP$TuD-)^y5z$a4ktQTV%rg0|eSIdBiSkI2#VnKh zEwrCr95t_PfHFB}MdpQ+iC)}M)O2o*8-C=TIEe&_S|*I;87O?kpN^W_k<{kP#7rXu zS2=#-@=7R^U2{mDNM$0~O%}@};=`|(AkGUOb6|JTk*g_tyv_C>0n4!DenE~xhoEXgKy@M zc_9v7=ynS4Zr)?Ly>OsUawS2cI>=aFOm(j+IM9DDC8^Cjc>8u3uCnBh@lSBj&WGfQ z`!Sv3{T!H+%%{8u$O7_Y(Q z;GI0$pPu_|SH@plMud}67anYLZU4g-Hf znaG=v3^B_j$G~w1l*tqelEo|&^~T4zVY5!F2SJ%=Sdn=lW#VFS9yN7ayk{Vm$wU$) zYMC&W_vG856;LMYY)NYKW#TYD8dq83I5HW^#C8tJ6Rk`hXkeL)zw&7>h!f&WB00sW z3CDeq`cixY%4Eo5k|JK2jJNmP1PA(pS8F?mHaU2p2bY5n(rJHsu36c`4jioeOhSY? zxJrU{pqCrW!CyVE_(76d+KvzByG7_%I znEfP!^O_{B$7`}MX!|`V6G^j>cYKh#o?K97!o3^OxpCNl}T=w zknwP!S5G2IEh>}wF*c!asb!{3 zCXO@eaKko+>IXxaY+6F*g_Mcel541`URiA|l!=EI2@*bVv6e(~ij%q#_o4f-s|Vmfe-cAd#48ii zhUE$1VDX+K1z}AG`jy^X4zB7&dr^I2q4f%I&@zRD2y<}tYMRZhjf>uagZqw=q!u}t z_jqAFI9Ps~B(ghrcpc5bEV~Pxz`^xrNrspXO5Z(r>;NG_+MLY-9zUWxNf&SR1S~k4N!Kl8> z1HIEY+RN!#KS%w5JAC*`GR}UW>t3|Q13gE6xHMdo>k%Y_^O{)DzGGEm{{sIxkGsp- zZRUd`6Wm-pDRx>r4a&rD3&{|(OeE&#;1^TRC6X*=nJ87XMY~k9Z+C$*S+S4I3n`QL z8~(+hus7$fLzy@nCPAW>31fLloy|+2OjI&RYV&1sAuJJBX}E8H4=9u16C_WhGRfFJ z0?TB;(TUa|PFgOBwa7t(GY9U0gHJAyM0N*v?58<+>G8s! z;NXHQBtuLGBcG032rsH#ZjvmfgB4D+$B6}Zme&9Wf8QtbLLAK0e1>=Th7ND%frEpe zkRVYVWGwIYuglND!Lk=5wRs0K$M42f=6hYp1P9gMkvx$cEVUhr4z6E3{xmok@|8q# zI_P~69kf%Zd;$*E{vavhIT+Y>Y#(sYwbMR>wM`CQAHd~cOf2nK|MzB&6gX(2MM8u* zsQ!ZHVA07#mEfRsf0EQ92WLv&jspkV4kd}~4#r%iIXHB)9zGK(br{JI(?OL_k7dC@ z+Yuy->7f5tWxOfpP3}Dk9Bi;8^Fkc7IZ=Z_m7_1V2M60tAVH!!$XMP0kMsAy!2vcT zwRs1_RcPNjJ*T_=5;(YT7ReLI!P{Rap@U9S)N;W=H763u>0nSAI@s#Fr2#m&XCX-u z&%u`uIyK;nYSYf2GH{9V7mSmj$MYVQiGQOyem~D^0LLE1e zWN==Ss&u?2trIMBpiEXL$4W;ul}U~^SDBGy#HJ@*pH5VK6S_9_X0GLgSZvY2JE zH(V7vK*nK;8I;NUTV!5HndFT8h(T}Fjx2&QF?vXXL@g7>^12kCuYxl9R7FyoFO!@1 z4&o}`jQE@eWn%S;BXuN2yfP`) z*ii%y&bS-(fbZaT9WDpcJZYbjO;?%T7aSZ`MM8u*xML0NK;N|Y*ivwC$c_2fAM0Jp{Jo)88zTn_|C6d~_gU-8CaFtS_%EsW}?~Wu- zBnMMFIiQ1moPQL9gA(0HB&UNpIq2Z&ThiOX!LvO{ig*r=^nP9o2l{mV*spvCKj|o-K{CWF6DP?>PH>-&K?##SVfQ=ES1Q5=_t86Hq3MUzO1cx9sd@a@DRqL7@{63V+!P(NZ zcj!^$9M*z^`CcSMn1h~lW^96Ys!%nNaF{)$#s@Cc6V**|`P zgUN{`NK^+I%Uk_=HU8Oa={}O$yo08LG7)!_f8tkgaC$Py6UjmAZ|>+|wA4HNa(K>B z63OXcVF^0;BYs90I5_?kNfFP%4xRI#!GUgbcYHG6!D?ORfj(*r?RhLoPaC7*4!`+? zjI$r;c~x0>p!@jbUV>h_@rq<{UK7nLcugdJMJ7U-$gL=y8r4)LxAnNnlY=lO!{WNT?}OslSzU^ zEfdD_lB7~{;6T^TBB{-niPyIiSSGo{Z7iTn;?I&ik;>$FcYiDs_wZl%EC`7UB$88{ zq-!Y7qvULS9!uk8k|JK2ObiKa3l7fQ-m7PHlY_+uTn?&?rI|By+`bBM@Lm!L5$52S zCA1{(qF0)5pf5=!NiA})WYWHw;Nas-lF06$Rs_w#CtCM+f`idnBtuLGM<4YK1P3kh zNfy(=U5>hVQ>KsYISm|qTTJGKIB2dUgAQ)|m{SZ6Dqki+qB_V}-s2ZbrhtPdD@bbd z4i*$-A#Q5XhsWTc#yyfJl7rToE73uh@>2)l@QQs*A~_vYyNeFK7@3p>2YTlhBt<+2 zJ$C=;g9o~mj3eK{`-aQ|J+jX?{HuH(CYE@0K^5*S)qXcE*5Jgg(FOx^>Xg{u7 zv8FK+%A_ogNIT&F4UgS@E?4KK6utCW4I=JB1i`3HF2W-3M#+9DG$md|M#Swe2^C= zTxAlWM*AK5ymwuXK$(n^o=k-ZE0a-av=8s)f-irCGP$itl3G+I!8ep$;ZhfMB#G>0 zlA%K@laZ!InNTK6)JcYzWpb!@(Mc$i`W_^UStek~F_piDj)lGNtQq_0{L;(9!tZVP2%Kb+)=R3>+WW3Wud z``E4paSm9LNKSG3y~Q#)XEC`Kl*zaWBt^V3X`^P~0S*qXZ{@MRsZ4UrxEzd%q&aB! zV$FPT(6m)+DnyurM-S2*47(Xq1`ghlAxSNAaKi&@d=|ugC6dVQ;8%5;gZs~>+k%4& zRY``J4sML>^Bf#B?MAYg4nEJY#+%Z@#4rLJlHnT z8Ou|MNjwe?t{*~Dn|ILU=|#ltBOkXK9DHR+@xEyR}Mf)DhpZC!!;9${15+cmO<25t~ zgG#$^0|#B7k)#$mc%tUhHaG_dy(Wq54hGoL94rj`6bcR&e;q9GpLC3pzOc>Xaqm;Pg%;lGDL{-_gN2o9`Zlb5No?NfFOM z30`!^?goAkn$_Fg$;o^usBjkt=|zRHha RXMUU`DdO$SqO=;D{{vDC>x}>a literal 0 HcmV?d00001 diff --git a/logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/trial_1/flags_0.json b/logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/trial_1/flags_0.json new file mode 100644 index 00000000..0f3ae243 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/trial_1/flags_0.json @@ -0,0 +1,62 @@ +{ + "logtostderr": false, + "alsologtostderr": false, + "log_dir": "", + "v": 0, + "verbosity": 0, + "logger_levels": {}, + "stderrthreshold": "fatal", + "showprefixforinfo": true, + "run_with_pdb": false, + "pdb_post_mortem": false, + "pdb": false, + "run_with_profiling": false, + "profile_file": null, + "use_cprofile_for_profiling": true, + "only_check_args": false, + "op_conversion_fallback_to_while_loop": true, + "delta_threshold": 0.5, + "tt_check_filter": false, + "tt_single_core_summaries": false, + "runtime_oom_exit": true, + "hbm_oom_exit": true, + "test_srcdir": "", + "test_tmpdir": "/tmp/absl_testing", + "test_random_seed": 301, + "test_randomize_ordering_seed": "", + "xml_output_file": "", + "experimental_orbax_use_distributed_process_id": false, + "experimental_orbax_use_distributed_barrier": false, + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "workload": "criteo1tb", + "tuning_ruleset": "self", + "tuning_search_space": null, + "num_tuning_trials": 1, + "data_dir": "/data/criteo1tb", + "imagenet_v2_data_dir": null, + "librispeech_tokenizer_vocab_path": "", + "framework": "pytorch", + "torch_compile": true, + "experiment_dir": "/experiment_runs", + "experiment_name": "submissions_a100/ademamix_golden/study_1", + "save_checkpoints": false, + "save_intermediate_checkpoints": true, + "resume_last_run": null, + "append_timestamp": false, + "use_wandb": false, + "profile": false, + "max_global_steps": null, + "overwrite": true, + "hparam_start_index": null, + "hparam_end_index": null, + "rng_seed": 1213513687, + "set_pytorch_max_split_size": false, + "pytorch_eval_num_workers": 0, + "capture_jax_trace": false, + "skip_evals": false, + "?": false, + "help": false, + "helpshort": false, + "helpfull": false, + "helpxml": false +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/trial_1/measurements.csv b/logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/trial_1/measurements.csv new file mode 100644 index 00000000..e6846a70 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/trial_1/measurements.csv @@ -0,0 +1,157 @@ +global_step,grad_norm,loss,train/loss,validation/loss,validation/num_examples,test/loss,test/num_examples,score,total_duration,accumulated_submission_time,accumulated_eval_time,accumulated_logging_time,preemption_count +0,0.4999999403953552,1.344266653060913,,,,,,,,,,, +1,,,1.3445777583756644,1.3570842317910403,83274637.0,1.3495619400801808,95000000.0,9.658968210220335,1870.628292798996,9.658968210220335,1860.652927160263,0.0,0.0 +1,0.4999999403953552,1.3443957567214966,,,,,,,,,,, +2,0.4999999403953552,1.3380703926086426,,,,,,,,,,, +3,0.4999999105930328,1.32061767578125,,,,,,,,,,, +4,0.4999999701976776,1.299688696861267,,,,,,,,,,, +5,0.4999999403953552,1.2685283422470093,,,,,,,,,,, +6,0.4999999403953552,1.2317031621932983,,,,,,,,,,, +7,0.4999999701976776,1.1889187097549438,,,,,,,,,,, +8,0.4999998807907104,1.1378042697906494,,,,,,,,,,, +9,0.4999999403953552,1.080533742904663,,,,,,,,,,, +10,0.4999999105930328,1.0173650979995728,,,,,,,,,,, +11,0.4999999403953552,0.9473499059677124,,,,,,,,,,, +12,0.4999999403953552,0.8734811544418335,,,,,,,,,,, +13,0.4999999403953552,0.7973098754882812,,,,,,,,,,, +14,0.4999999701976776,0.7207428216934204,,,,,,,,,,, +15,0.4999999105930328,0.6443389654159546,,,,,,,,,,, +16,0.4999999105930328,0.5689073801040649,,,,,,,,,,, +17,0.4999999105930328,0.4975816309452057,,,,,,,,,,, +18,0.4999999105930328,0.431009441614151,,,,,,,,,,, +19,0.4999998807907104,0.3701639771461487,,,,,,,,,,, +20,0.4999998211860657,0.3207697868347168,,,,,,,,,,, +21,0.4999997615814209,0.2850112318992615,,,,,,,,,,, +22,0.4999996721744537,0.2525757551193237,,,,,,,,,,, +23,0.4999993145465851,0.236987829208374,,,,,,,,,,, +24,0.3907130062580108,0.2246416807174682,,,,,,,,,,, +25,0.4420184791088104,0.2187784016132354,,,,,,,,,,, +26,0.4999992251396179,0.2230259031057357,,,,,,,,,,, +27,0.4999994039535522,0.226404383778572,,,,,,,,,,, +28,0.4999993741512298,0.2211626917123794,,,,,,,,,,, +29,0.4999992847442627,0.2129182964563369,,,,,,,,,,, +30,0.499999076128006,0.202856034040451,,,,,,,,,,, +31,0.3714069724082947,0.1987889111042022,,,,,,,,,,, +32,0.2478403449058532,0.1920698136091232,,,,,,,,,,, +33,0.2909159064292907,0.1918447017669677,,,,,,,,,,, +34,0.3026428520679474,0.188725396990776,,,,,,,,,,, +35,0.2326929867267608,0.1812068819999694,,,,,,,,,,, +36,0.1770111471414566,0.179976612329483,,,,,,,,,,, +37,0.2500056326389313,0.1747719794511795,,,,,,,,,,, +38,0.1876603811979293,0.1705579161643982,,,,,,,,,,, +39,0.15771484375,0.1650077402591705,,,,,,,,,,, +40,0.1332434713840484,0.1625821441411972,,,,,,,,,,, +41,0.120769664645195,0.160823568701744,,,,,,,,,,, +42,0.1046208515763282,0.1570714116096496,,,,,,,,,,, +43,0.0982748642563819,0.1543671190738678,,,,,,,,,,, +44,0.0907251685857772,0.1492833644151687,,,,,,,,,,, +45,0.1201491355895996,0.1496166586875915,,,,,,,,,,, +46,0.116980530321598,0.1493081003427505,,,,,,,,,,, +47,0.0750320702791214,0.1475882828235626,,,,,,,,,,, +48,0.0871409624814987,0.1454977989196777,,,,,,,,,,, +49,0.0745422169566154,0.1446480751037597,,,,,,,,,,, +50,0.1023278385400772,0.144313633441925,,,,,,,,,,, +51,0.0892822965979576,0.1420854479074478,,,,,,,,,,, +52,0.0588252246379852,0.1383918821811676,,,,,,,,,,, +53,0.1551447510719299,0.1386710405349731,,,,,,,,,,, +54,0.1076757237315177,0.1381655633449554,,,,,,,,,,, +55,0.0469033382833004,0.1371026933193206,,,,,,,,,,, +56,0.1505993604660034,0.1365167498588562,,,,,,,,,,, +57,0.1853561252355575,0.1367435753345489,,,,,,,,,,, +58,0.1466805040836334,0.1348283588886261,,,,,,,,,,, +59,0.0477874055504798,0.136686623096466,,,,,,,,,,, +60,0.139274850487709,0.1334722638130188,,,,,,,,,,, +61,0.3310624063014984,0.1352578401565551,,,,,,,,,,, +62,0.499999076128006,0.1332001239061355,,,,,,,,,,, +63,0.4107071161270141,0.1336231678724289,,,,,,,,,,, +64,0.0206294152885675,0.1324027329683303,,,,,,,,,,, +65,0.4164749383926391,0.1327193230390548,,,,,,,,,,, +66,0.4544623792171478,0.1334672421216964,,,,,,,,,,, +67,0.02548205293715,0.1320918798446655,,,,,,,,,,, +68,0.4466593265533447,0.133773997426033,,,,,,,,,,, +69,0.3971154689788818,0.1310182511806488,,,,,,,,,,, +70,0.1217448189854621,0.1312794685363769,,,,,,,,,,, +71,0.499999076128006,0.1328619420528412,,,,,,,,,,, +72,0.3646708726882934,0.1332804858684539,,,,,,,,,,, +73,0.1177696362137794,0.1310909539461136,,,,,,,,,,, +74,0.4433577060699463,0.1323015391826629,,,,,,,,,,, +75,0.3802143335342407,0.1347779333591461,,,,,,,,,,, +76,0.0600906163454055,0.1361371129751205,,,,,,,,,,, +77,0.4754917919635772,0.138260617852211,,,,,,,,,,, +78,0.4808640778064728,0.1403313875198364,,,,,,,,,,, +79,0.041332833468914,0.1398908793926239,,,,,,,,,,, +80,0.4551554024219513,0.1416103690862655,,,,,,,,,,, +81,0.4652088284492492,0.1391115188598632,,,,,,,,,,, +82,0.0405144803225994,0.1389724612236023,,,,,,,,,,, +83,0.4999991655349731,0.1374754309654235,,,,,,,,,,, +84,0.406187891960144,0.1383841335773468,,,,,,,,,,, +85,0.1090170368552208,0.1368868201971054,,,,,,,,,,, +86,0.4999990165233612,0.1380661725997924,,,,,,,,,,, +87,0.3887943923473358,0.1365829408168792,,,,,,,,,,, +88,0.1239143013954162,0.135954812169075,,,,,,,,,,, +89,0.4999990463256836,0.1374380737543106,,,,,,,,,,, +90,0.3447398841381073,0.134689375758171,,,,,,,,,,, +91,0.1647352874279022,0.1349323391914367,,,,,,,,,,, +92,0.4999991357326507,0.135207325220108,,,,,,,,,,, +93,0.3629132211208343,0.1360347121953964,,,,,,,,,,, +94,0.0733061283826828,0.1337140947580337,,,,,,,,,,, +95,0.4768237471580505,0.1318236887454986,,,,,,,,,,, +96,0.4111478626728058,0.1299033164978027,,,,,,,,,,, +97,0.0902777314186096,0.1281714141368866,,,,,,,,,,, +98,0.4347357153892517,0.1311616450548172,,,,,,,,,,, +99,0.29989093542099,0.1300349980592727,,,,,,,,,,, +100,0.1024882048368454,0.1295927464962005,,,,,,,,,,, +200,,,0.1268694460864347,0.1289985184971938,83274637.0,0.1314415748999344,95000000.0,366.55974078178406,3281.604463815689,366.55974078178406,2913.9182076454163,0.0563101768493652,0.0 +402,,,0.1292311273234526,0.1279783523133029,83274637.0,0.1304406822643079,95000000.0,722.8921821117401,4687.963960409164,722.8921821117401,3963.20534324646,0.0872905254364013,0.0 +500,0.0453804321587085,0.1280699372291565,,,,,,,,,,, +601,,,0.128460360749033,0.1267814876495373,83274637.0,0.1291329746458354,95000000.0,1079.2291326522827,6090.243111848831,1079.2291326522827,5008.411679267883,0.1173408031463623,0.0 +802,,,0.1261564336058258,0.1262946168819717,83274637.0,0.1287239919322766,95000000.0,1435.5280559062958,7512.024815559387,1435.5280559062958,6073.168746471405,0.1463634967803955,0.0 +1000,0.0173730254173278,0.1210454404354095,,,,,,,,,,, +1008,,,0.1273649113555809,0.126224402964769,83274637.0,0.1285700634242007,95000000.0,1792.1379549503326,8898.164870262146,1792.1379549503326,7101.967989206314,0.1746838092803955,0.0 +1220,,,0.1249698756503415,0.1257965216111919,83274637.0,0.1281569895374499,95000000.0,2148.3401172161102,10276.384850740433,2148.3401172161102,8123.249813318253,0.2063307762145996,0.0 +1434,,,0.1262998824987502,0.125697069969302,83274637.0,0.1278985934318141,95000000.0,2504.3137986660004,11643.921679496763,2504.3137986660004,9134.081369876862,0.235431432723999,0.0 +1500,0.0497593842446804,0.1278442740440368,,,,,,,,,,, +1636,,,0.1236481840879651,0.1255930255676328,83274637.0,0.1279189863202546,95000000.0,2860.8038444519043,12982.689307451248,2860.8038444519043,10115.627338647842,0.2652029991149902,0.0 +1846,,,0.1225691066353856,0.1255440421935849,83274637.0,0.1279417368135151,95000000.0,3217.747561454773,14279.446210861206,3217.747561454773,11054.704303264618,0.2939579486846924,0.0 +2000,0.0585377253592014,0.1216020137071609,,,,,,,,,,, +2059,,,0.1235477273606574,0.1253004452267213,83274637.0,0.1278243510589599,95000000.0,3573.622041463852,15540.881484270096,3573.622041463852,11959.5324447155,0.3234055042266845,0.0 +2264,,,0.1214590713949025,0.1251349367273924,83274637.0,0.1276238236686305,95000000.0,3929.251423597336,16710.97428059578,3929.251423597336,12773.260489463806,0.3537471294403076,0.0 +2470,,,0.1235114090716656,0.1252825022753245,83274637.0,0.127745887270315,95000000.0,4285.6617822647095,17808.054710149765,4285.6617822647095,13513.194631576538,0.3836984634399414,0.0 +2500,0.0070999697782099,0.1168490126729011,,,,,,,,,,, +2695,,,0.1228290405714404,0.1249627153190607,83274637.0,0.1273950589214124,95000000.0,4642.696635723114,18823.92814230919,4642.696635723114,14171.295335292816,0.4115707874298095,0.0 +2915,,,0.1245766903279074,0.1247602831126512,83274637.0,0.1272692146180805,95000000.0,4998.295198917389,19694.56010222435,4998.295198917389,14685.586983680723,0.4423439502716064,0.0 +3000,0.0076904115267097,0.124981939792633,,,,,,,,,,, +3144,,,0.1250313014233972,0.1247420626848063,83274637.0,0.1272691373829088,95000000.0,5354.927243947983,20535.125536441803,5354.927243947983,15168.773005723951,0.4729907512664795,0.0 +3366,,,0.1235264876040958,0.1248940205799324,83274637.0,0.127170024838578,95000000.0,5710.522954463959,21374.258639097214,5710.522954463959,15651.562380552292,0.5058600902557373,0.0 +3500,0.0061207166872918,0.1138783395290374,,,,,,,,,,, +3588,,,0.1222109906846291,0.1247722997757567,83274637.0,0.127062874858816,95000000.0,6066.370519399643,22215.099447250366,6066.370519399643,16135.808561325071,0.5355989933013916,0.0 +3792,,,0.1243355281566004,0.1245174247297344,83274637.0,0.126931378600753,95000000.0,6422.8106808662415,23054.88243794441,6422.8106808662415,16618.414626598358,0.5651638507843018,0.0 +3988,,,0.1214501012929081,0.1245933773395324,83274637.0,0.1269691514418752,95000000.0,6780.02803850174,23897.213665246964,6780.02803850174,17102.797516345978,0.5944151878356934,0.0 +4000,0.0084276814013719,0.126094102859497,,,,,,,,,,, +4189,,,0.122534525138587,0.1244867060856099,83274637.0,0.1267508165151495,95000000.0,7135.8886506557465,24736.193083524704,7135.8886506557465,17585.180767774582,0.6244139671325684,0.0 +4386,,,0.1228863687585938,0.1245126588331126,83274637.0,0.1267406875634444,95000000.0,7493.290529727936,25576.752193450928,7493.290529727936,18067.606957674023,0.6540706157684326,0.0 +4500,0.0130800837650895,0.1234208643436431,,,,,,,,,,, +4597,,,0.1241817561561066,0.1245861310388407,83274637.0,0.1268185528223941,95000000.0,7850.796292304993,26420.616366624832,7850.796292304993,18553.23108625412,0.6840434074401855,0.0 +4811,,,0.1221399161260712,0.124266617257228,83274637.0,0.1264940008418033,95000000.0,8207.194816350937,27261.325538396835,8207.194816350937,19036.80064892769,0.7143452167510986,0.0 +5000,0.0129254329949617,0.122698001563549,,,,,,,,,,, +5029,,,0.1234449331014513,0.1242678982410396,83274637.0,0.1265891018561112,95000000.0,8564.014400959015,28102.96358895302,8564.014400959015,19520.883132219315,0.7445018291473389,0.0 +5230,,,0.1241965349891888,0.1242795916885101,83274637.0,0.126593013455682,95000000.0,8920.203042268753,28944.3425180912,8920.203042268753,20005.3422164917,0.7747159004211426,0.0 +5455,,,0.1228247421014758,0.1244604314047475,83274637.0,0.1267606357121517,95000000.0,9276.86423420906,29785.160937786102,9276.86423420906,20488.75330924988,0.8045017719268799,0.0 +5500,0.0203686580061912,0.1275954246520996,,,,,,,,,,, +5688,,,0.1206181872666861,0.1242689610430974,83274637.0,0.1265499985992431,95000000.0,9632.654585123062,30624.85894203186,9632.654585123062,20971.911014795303,0.8349716663360596,0.0 +5911,,,0.1222813890025916,0.1242744408357446,83274637.0,0.1266273077835886,95000000.0,9989.594222784042,31464.81358885765,9989.594222784042,21454.18979382515,0.8636422157287598,0.0 +6000,0.0057965260930359,0.124400794506073,,,,,,,,,,, +6181,,,0.1236555678991646,0.1243073543416901,83274637.0,0.1266926717426501,95000000.0,10345.668259859083,32304.610867500305,10345.668259859083,21937.16048359871,0.8935234546661377,0.0 +6392,,,0.1225726111097063,0.1246283052507866,83274637.0,0.127019188752345,95000000.0,10702.254277944565,33147.38299417496,10702.254277944565,22422.61490345001,0.9256329536437988,0.0 +6500,0.0055809887126088,0.11697818338871,,,,,,,,,,, +6612,,,0.1228522637471653,0.1242170637967608,83274637.0,0.1266071728648938,95000000.0,11059.34743642807,33989.80683994293,11059.34743642807,22907.217544555664,0.9574995040893556,0.0 +6851,,,0.1241570816219758,0.1242242216134974,83274637.0,0.1264844068511963,95000000.0,11415.9016726017,34831.24528861046,11415.9016726017,23391.370171785355,0.9887738227844238,0.0 +7000,0.0259947925806045,0.1201906651258468,,,,,,,,,,, +7067,,,0.1240802963011255,0.1242021702131752,83274637.0,0.126472298812545,95000000.0,11771.739629983902,35672.36800336838,11771.739629983902,23875.930784225464,1.0206894874572754,0.0 +7298,,,0.1208137100278091,0.124043713182727,83274637.0,0.1263520182782624,95000000.0,12129.126768112184,36514.85768580437,12129.126768112184,24360.304978609085,1.0505201816558838,0.0 +7500,0.0054081236012279,0.1202429309487342,,,,,,,,,,, +7525,,,0.1211405176038705,0.1238598991762789,83274637.0,0.1261545997556987,95000000.0,12485.66578912735,37355.89405155182,12485.66578912735,24844.07708263397,1.0818431377410889,0.0 +7736,,,0.1213503510685912,0.1238473428037413,83274637.0,0.1260869973560534,95000000.0,12843.221409320831,38198.704989910126,12843.221409320831,25328.61328649521,1.111832618713379,0.0 +7941,,,0.1222058667959075,0.1239789691314407,83274637.0,0.126294937829911,95000000.0,13199.509422063828,39039.16946601868,13199.509422063828,25812.071299791336,1.142986536026001,0.0 +8000,0.0066057760268449,0.1203106790781021,,,,,,,,,,, +8183,,,,,,,,13555.211057901382,,,,,0.0 diff --git a/logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/trial_1/meta_data_0.json b/logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/trial_1/meta_data_0.json new file mode 100644 index 00000000..1e351b7e --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/criteo1tb_pytorch/trial_1/meta_data_0.json @@ -0,0 +1,72 @@ +{ + "workload.embed_dim": 128, + "workload.eval_batch_size": 8192, + "workload.eval_period_time_sec": 356, + "workload.max_allowed_runtime_sec": 8915, + "workload.num_dense_features": 13, + "workload.num_eval_train_examples": 83279872, + "workload.num_test_examples": 95000000, + "workload.num_train_examples": 4195197692, + "workload.num_validation_examples": 83274637, + "workload.step_hint": 10666, + "workload.target_metric_name": "loss", + "workload.test_target_value": 0.126041, + "workload.train_mean": 0.126041, + "workload.train_stddev": 0.126041, + "workload.use_layer_norm": false, + "workload.use_resnet": false, + "workload.validation_target_value": 0.123735, + "workload.vocab_size": 4194304, + "cpu.util.avg_percent_since_last": 5.8, + "cpu.freq.current": 2200.1639999999993, + "mem.total": 359053524992, + "mem.available": 349788700672, + "mem.used": 6108475392, + "mem.percent_used": 2.6, + "mem.read_bytes_since_boot": 47576762065920, + "mem.write_bytes_since_boot": 123288568832, + "net.bytes_sent_since_boot": 31799591, + "net.bytes_recv_since_boot": 31802693, + "gpu.count": 4, + "gpu.0.compute.util": 0.08, + "gpu.0.mem.util": 0.1278076171875, + "gpu.0.mem.total": 40960.0, + "gpu.0.mem.used": 5235.0, + "gpu.0.mem.free": 35092.0, + "gpu.0.temp.current": 34.0, + "gpu.1.compute.util": 0.08, + "gpu.1.mem.util": 0.1278076171875, + "gpu.1.mem.total": 40960.0, + "gpu.1.mem.used": 5235.0, + "gpu.1.mem.free": 35092.0, + "gpu.1.temp.current": 36.0, + "gpu.2.compute.util": 0.06, + "gpu.2.mem.util": 0.1278076171875, + "gpu.2.mem.total": 40960.0, + "gpu.2.mem.used": 5235.0, + "gpu.2.mem.free": 35092.0, + "gpu.2.temp.current": 34.0, + "gpu.3.compute.util": 0.08, + "gpu.3.mem.util": 0.1278076171875, + "gpu.3.mem.total": 40960.0, + "gpu.3.mem.used": 5235.0, + "gpu.3.mem.free": 35092.0, + "gpu.3.temp.current": 36.0, + "gpu.avg.compute.util": 0.075, + "gpu.avg.mem.util": 0.1278076171875, + "gpu.avg.mem.total": 40960.0, + "gpu.avg.mem.used": 5235.0, + "gpu.avg.mem.free": 35092.0, + "gpu.avg.temp.current": 35.0, + "os_platform": "Linux-6.1.0-44-cloud-amd64-x86_64-with-glibc2.31", + "python_version": "3.11.10", + "python_compiler": "GCC 9.4.0", + "git_branch": "main", + "git_commit_hash": "b21be29be0a1573fb4f78f849aea019cdb520862", + "cpu_model_name": "Intel(R) Xeon(R) CPU @ 2.20GHz", + "cpu_count": 24, + "gpu_model_name": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_driver": "550.90.12", + "rng_seed": 1213513687 +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/fastmri_pytorch_09-12-2026-09-41-29.log b/logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/fastmri_pytorch_09-12-2026-09-41-29.log new file mode 100644 index 00000000..65e39541 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/fastmri_pytorch_09-12-2026-09-41-29.log @@ -0,0 +1,443 @@ +torchrun --redirects 1:0,2:0,3:0 --standalone --nnodes=1 --nproc_per_node=4 submission_runner.py --framework=pytorch --workload=fastmri --submission_path=submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py --data_dir=/data/fastmri --experiment_dir=/experiment_runs --experiment_name=submissions_a100/ademamix_golden/study_1 --overwrite=True --save_checkpoints=False --rng_seed=-638637756 --torch_compile=true --tuning_ruleset=self 2>&1 | tee -a /logs/fastmri_pytorch_09-12-2026-09-41-29.log +W0912 09:41:55.248000 9 site-packages/torch/distributed/run.py:803] +W0912 09:41:55.248000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0912 09:41:55.248000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0912 09:41:55.248000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-12 09:42:10.335757: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-12 09:42:10.335757: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-12 09:42:10.335793: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-12 09:42:10.335757: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789206130.857158 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789206130.857157 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789206130.857148 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789206130.857172 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789206130.950851 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789206130.950861 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789206130.950860 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789206130.950869 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789206132.303809 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789206132.303810 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789206132.303846 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789206132.303849 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789206132.303851 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789206132.303852 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789206132.303855 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789206132.303857 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789206132.303828 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789206132.303861 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789206132.303864 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789206132.303866 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789206132.303847 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789206132.303876 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789206132.303878 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789206132.303880 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789206159.708917 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789206159.708911 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789206159.808207 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789206159.821891 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W912 09:42:58.047565395 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0912 09:43:00.675432 139836003767488 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/fastmri_pytorch. +I0912 09:43:00.675433 140260576244928 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/fastmri_pytorch. +I0912 09:43:00.675436 140461562307776 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/fastmri_pytorch. +I0912 09:43:00.675468 140373236487360 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/fastmri_pytorch. +I0912 09:43:00.892448 139836003767488 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/fastmri_pytorch/trial_1. +I0912 09:43:01.205085 139836003767488 submission_runner.py:242] Initializing dataset. +I0912 09:43:01.205259 139836003767488 submission_runner.py:251] Initializing model. +I0912 09:43:01.826714 139836003767488 submission_runner.py:290] Performing `torch.compile`. +I0912 09:43:05.696992 139836003767488 submission_runner.py:294] Initializing optimizer. +I0912 09:43:05.697635 139836003767488 submission_runner.py:299] Initializing metrics bundle. +I0912 09:43:05.697797 139836003767488 submission_runner.py:321] Initializing checkpoint and logger. +I0912 09:43:05.698643 139836003767488 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_1/fastmri_pytorch/trial_1/meta_data_0.json. +I0912 09:43:05.698707 140373236487360 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0912 09:43:05.698709 140260576244928 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0912 09:43:05.698726 140461562307776 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0912 09:43:05.698830 139836003767488 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0912 09:43:05.698864 140260576244928 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0912 09:43:05.698870 140373236487360 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0912 09:43:05.698883 139836003767488 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0912 09:43:05.698881 140461562307776 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0912 09:43:06.157143 139836003767488 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_1/fastmri_pytorch/trial_1/flags_0.json. +I0912 09:43:06.246780 139836003767488 submission_runner.py:359] Starting training loop. +[rank2]:W0912 09:43:06.395000 40 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank3]:W0912 09:43:06.395000 41 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank1]:W0912 09:43:06.395000 39 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +WARNING:tensorflow:AutoGraph could not transform and will run it as-is. +Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. +Cause: (, (leaf_jax_array := getattr(leaf, '__jax_array__', None))) +To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert +W0912 09:43:08.232978 139836003767488 ag_logging.py:142] AutoGraph could not transform and will run it as-is. +Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. +Cause: (, (leaf_jax_array := getattr(leaf, '__jax_array__', None))) +To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert +WARNING:tensorflow:AutoGraph could not transform and will run it as-is. +Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. +Cause: (, (aval := get_aval(a))) +To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert +W0912 09:43:11.308917 139836003767488 ag_logging.py:142] AutoGraph could not transform and will run it as-is. +Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. +Cause: (, (aval := get_aval(a))) +To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert +[rank0]:W0912 09:46:31.491000 38 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +I0912 09:47:42.921128 139793853392640 logging_writer.py:48] [0] global_step=0, grad_norm=0.5, loss=0.942922 +I0912 09:47:43.215552 139836003767488 submission.py:307] 0) loss = 0.943, grad_norm = 0.500 +I0912 09:47:43.833606 139836003767488 spec.py:333] Evaluating on the training split. +[rank3]:W0912 09:50:42.567000 41 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] torch._dynamo hit config.recompile_limit (8) +[rank3]:W0912 09:50:42.567000 41 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] function: 'forward' (/algorithmic-efficiency/algoperf/workloads/fastmri/fastmri_pytorch/models.py:139) +[rank3]:W0912 09:50:42.567000 41 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] last reason: 5/7: GLOBAL_STATE changed: grad_mode +[rank3]:W0912 09:50:42.567000 41 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To log all recompilation reasons, use TORCH_LOGS="recompiles". +[rank3]:W0912 09:50:42.567000 41 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To diagnose recompilation issues, see https://pytorch.org/docs/main/torch.compiler_troubleshooting.html +[rank0]:W0912 09:50:42.571000 38 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] torch._dynamo hit config.recompile_limit (8) +[rank0]:W0912 09:50:42.571000 38 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] function: 'forward' (/algorithmic-efficiency/algoperf/workloads/fastmri/fastmri_pytorch/models.py:139) +[rank0]:W0912 09:50:42.571000 38 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] last reason: 5/7: GLOBAL_STATE changed: grad_mode +[rank0]:W0912 09:50:42.571000 38 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To log all recompilation reasons, use TORCH_LOGS="recompiles". +[rank0]:W0912 09:50:42.571000 38 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To diagnose recompilation issues, see https://pytorch.org/docs/main/torch.compiler_troubleshooting.html +[rank2]:W0912 09:50:42.606000 40 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] torch._dynamo hit config.recompile_limit (8) +[rank2]:W0912 09:50:42.606000 40 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] function: 'forward' (/algorithmic-efficiency/algoperf/workloads/fastmri/fastmri_pytorch/models.py:139) +[rank2]:W0912 09:50:42.606000 40 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] last reason: 5/7: GLOBAL_STATE changed: grad_mode +[rank2]:W0912 09:50:42.606000 40 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To log all recompilation reasons, use TORCH_LOGS="recompiles". +[rank2]:W0912 09:50:42.606000 40 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To diagnose recompilation issues, see https://pytorch.org/docs/main/torch.compiler_troubleshooting.html +[rank1]:W0912 09:50:42.606000 39 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] torch._dynamo hit config.recompile_limit (8) +[rank1]:W0912 09:50:42.606000 39 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] function: 'forward' (/algorithmic-efficiency/algoperf/workloads/fastmri/fastmri_pytorch/models.py:139) +[rank1]:W0912 09:50:42.606000 39 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] last reason: 5/7: GLOBAL_STATE changed: grad_mode +[rank1]:W0912 09:50:42.606000 39 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To log all recompilation reasons, use TORCH_LOGS="recompiles". +[rank1]:W0912 09:50:42.606000 39 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To diagnose recompilation issues, see https://pytorch.org/docs/main/torch.compiler_troubleshooting.html +I0912 09:53:37.589074 139836003767488 spec.py:346] Evaluating on the validation split. +I0912 09:56:58.068164 139836003767488 spec.py:363] Evaluating on the test split. +I0912 10:00:12.895488 139836003767488 submission_runner.py:516] Time since start: 1026.65s, Step: 1, {'train/ssim': 0.2550915649959019, 'train/loss': 0.9745864868164062, 'validation/ssim': 0.2467691210058209, 'validation/loss': 0.9774193278040588, 'validation/num_examples': 3554, 'test/ssim': 0.2693324979874249, 'test/loss': 0.976249432770176, 'test/num_examples': 3581, 'score': 276.97007966041565, 'total_duration': 1026.6488287448883, 'accumulated_submission_time': 276.97007966041565, 'accumulated_eval_time': 749.0619850158691, 'accumulated_logging_time': 0} +I0912 10:00:12.991567 139762286749440 logging_writer.py:48] [1] accumulated_eval_time=749.062, accumulated_logging_time=0, accumulated_submission_time=276.97, global_step=1, preemption_count=0, score=276.97, test/loss=0.976249, test/num_examples=3581, test/ssim=0.269332, total_duration=1026.65, train/loss=0.974586, train/ssim=0.255092, validation/loss=0.977419, validation/num_examples=3554, validation/ssim=0.246769 +I0912 10:00:13.688950 139762278356736 logging_writer.py:48] [1] global_step=1, grad_norm=0.5, loss=0.963639 +I0912 10:00:13.692503 139836003767488 submission.py:307] 1) loss = 0.964, grad_norm = 0.500 +I0912 10:00:13.784742 139762286749440 logging_writer.py:48] [2] global_step=2, grad_norm=0.5, loss=0.986867 +I0912 10:00:13.788482 139836003767488 submission.py:307] 2) loss = 0.987, grad_norm = 0.500 +I0912 10:00:13.867385 139762278356736 logging_writer.py:48] [3] global_step=3, grad_norm=0.5, loss=1.01453 +I0912 10:00:13.872540 139836003767488 submission.py:307] 3) loss = 1.015, grad_norm = 0.500 +I0912 10:00:13.951164 139762286749440 logging_writer.py:48] [4] global_step=4, grad_norm=0.5, loss=0.984116 +I0912 10:00:13.955748 139836003767488 submission.py:307] 4) loss = 0.984, grad_norm = 0.500 +I0912 10:00:14.042870 139762278356736 logging_writer.py:48] [5] global_step=5, grad_norm=0.5, loss=0.926645 +I0912 10:00:14.047575 139836003767488 submission.py:307] 5) loss = 0.927, grad_norm = 0.500 +I0912 10:00:14.130165 139762286749440 logging_writer.py:48] [6] global_step=6, grad_norm=0.5, loss=0.913222 +I0912 10:00:14.134351 139836003767488 submission.py:307] 6) loss = 0.913, grad_norm = 0.500 +I0912 10:00:14.225639 139762278356736 logging_writer.py:48] [7] global_step=7, grad_norm=0.5, loss=1.02795 +I0912 10:00:14.229414 139836003767488 submission.py:307] 7) loss = 1.028, grad_norm = 0.500 +I0912 10:00:14.315486 139762286749440 logging_writer.py:48] [8] global_step=8, grad_norm=0.5, loss=1.02894 +I0912 10:00:14.319140 139836003767488 submission.py:307] 8) loss = 1.029, grad_norm = 0.500 +I0912 10:00:14.406059 139762278356736 logging_writer.py:48] [9] global_step=9, grad_norm=0.5, loss=0.939941 +I0912 10:00:14.410448 139836003767488 submission.py:307] 9) loss = 0.940, grad_norm = 0.500 +I0912 10:00:14.495194 139762286749440 logging_writer.py:48] [10] global_step=10, grad_norm=0.5, loss=0.875063 +I0912 10:00:14.499793 139836003767488 submission.py:307] 10) loss = 0.875, grad_norm = 0.500 +I0912 10:00:14.585864 139762278356736 logging_writer.py:48] [11] global_step=11, grad_norm=0.5, loss=1.00473 +I0912 10:00:14.589501 139836003767488 submission.py:307] 11) loss = 1.005, grad_norm = 0.500 +I0912 10:00:14.672624 139762286749440 logging_writer.py:48] [12] global_step=12, grad_norm=0.5, loss=0.966425 +I0912 10:00:14.677422 139836003767488 submission.py:307] 12) loss = 0.966, grad_norm = 0.500 +I0912 10:00:14.762581 139762278356736 logging_writer.py:48] [13] global_step=13, grad_norm=0.5, loss=0.911942 +I0912 10:00:14.766144 139836003767488 submission.py:307] 13) loss = 0.912, grad_norm = 0.500 +I0912 10:00:14.846648 139762286749440 logging_writer.py:48] [14] global_step=14, grad_norm=0.5, loss=0.928037 +I0912 10:00:14.850346 139836003767488 submission.py:307] 14) loss = 0.928, grad_norm = 0.500 +I0912 10:00:14.936311 139762278356736 logging_writer.py:48] [15] global_step=15, grad_norm=0.5, loss=0.910811 +I0912 10:00:14.942224 139836003767488 submission.py:307] 15) loss = 0.911, grad_norm = 0.500 +I0912 10:00:15.018955 139762286749440 logging_writer.py:48] [16] global_step=16, grad_norm=0.5, loss=0.899606 +I0912 10:00:15.023697 139836003767488 submission.py:307] 16) loss = 0.900, grad_norm = 0.500 +I0912 10:00:15.106816 139762278356736 logging_writer.py:48] [17] global_step=17, grad_norm=0.5, loss=0.842915 +I0912 10:00:15.110900 139836003767488 submission.py:307] 17) loss = 0.843, grad_norm = 0.500 +I0912 10:00:15.194152 139762286749440 logging_writer.py:48] [18] global_step=18, grad_norm=0.5, loss=0.831488 +I0912 10:00:15.199828 139836003767488 submission.py:307] 18) loss = 0.831, grad_norm = 0.500 +I0912 10:00:15.286657 139762278356736 logging_writer.py:48] [19] global_step=19, grad_norm=0.5, loss=0.851235 +I0912 10:00:15.291600 139836003767488 submission.py:307] 19) loss = 0.851, grad_norm = 0.500 +I0912 10:00:15.377541 139762286749440 logging_writer.py:48] [20] global_step=20, grad_norm=0.5, loss=0.763819 +I0912 10:00:15.382030 139836003767488 submission.py:307] 20) loss = 0.764, grad_norm = 0.500 +I0912 10:00:15.463068 139762278356736 logging_writer.py:48] [21] global_step=21, grad_norm=0.5, loss=0.762511 +I0912 10:00:15.466597 139836003767488 submission.py:307] 21) loss = 0.763, grad_norm = 0.500 +I0912 10:00:15.553433 139762286749440 logging_writer.py:48] [22] global_step=22, grad_norm=0.5, loss=0.767869 +I0912 10:00:15.558103 139836003767488 submission.py:307] 22) loss = 0.768, grad_norm = 0.500 +I0912 10:00:15.634931 139762278356736 logging_writer.py:48] [23] global_step=23, grad_norm=0.5, loss=0.829193 +I0912 10:00:15.639906 139836003767488 submission.py:307] 23) loss = 0.829, grad_norm = 0.500 +I0912 10:00:15.717921 139762286749440 logging_writer.py:48] [24] global_step=24, grad_norm=0.5, loss=0.754285 +I0912 10:00:15.722727 139836003767488 submission.py:307] 24) loss = 0.754, grad_norm = 0.500 +I0912 10:00:15.806948 139762278356736 logging_writer.py:48] [25] global_step=25, grad_norm=0.5, loss=0.777798 +I0912 10:00:15.814419 139836003767488 submission.py:307] 25) loss = 0.778, grad_norm = 0.500 +I0912 10:00:15.882901 139762286749440 logging_writer.py:48] [26] global_step=26, grad_norm=0.5, loss=0.658879 +I0912 10:00:15.886679 139836003767488 submission.py:307] 26) loss = 0.659, grad_norm = 0.500 +I0912 10:00:15.962820 139762278356736 logging_writer.py:48] [27] global_step=27, grad_norm=0.5, loss=0.697294 +I0912 10:00:15.966467 139836003767488 submission.py:307] 27) loss = 0.697, grad_norm = 0.500 +I0912 10:00:16.049845 139762286749440 logging_writer.py:48] [28] global_step=28, grad_norm=0.5, loss=0.683869 +I0912 10:00:16.053375 139836003767488 submission.py:307] 28) loss = 0.684, grad_norm = 0.500 +I0912 10:00:16.132560 139762278356736 logging_writer.py:48] [29] global_step=29, grad_norm=0.5, loss=0.617089 +I0912 10:00:16.136384 139836003767488 submission.py:307] 29) loss = 0.617, grad_norm = 0.500 +I0912 10:00:16.213636 139762286749440 logging_writer.py:48] [30] global_step=30, grad_norm=0.5, loss=0.585415 +I0912 10:00:16.218350 139836003767488 submission.py:307] 30) loss = 0.585, grad_norm = 0.500 +I0912 10:00:16.294660 139762278356736 logging_writer.py:48] [31] global_step=31, grad_norm=0.5, loss=0.60804 +I0912 10:00:16.298506 139836003767488 submission.py:307] 31) loss = 0.608, grad_norm = 0.500 +I0912 10:00:16.380790 139762286749440 logging_writer.py:48] [32] global_step=32, grad_norm=0.5, loss=0.566089 +I0912 10:00:16.384872 139836003767488 submission.py:307] 32) loss = 0.566, grad_norm = 0.500 +I0912 10:00:16.460956 139762278356736 logging_writer.py:48] [33] global_step=33, grad_norm=0.5, loss=0.531561 +I0912 10:00:16.465698 139836003767488 submission.py:307] 33) loss = 0.532, grad_norm = 0.500 +I0912 10:00:16.543418 139762286749440 logging_writer.py:48] [34] global_step=34, grad_norm=0.5, loss=0.536388 +I0912 10:00:16.547669 139836003767488 submission.py:307] 34) loss = 0.536, grad_norm = 0.500 +I0912 10:00:16.624993 139762278356736 logging_writer.py:48] [35] global_step=35, grad_norm=0.5, loss=0.555533 +I0912 10:00:16.629102 139836003767488 submission.py:307] 35) loss = 0.556, grad_norm = 0.500 +I0912 10:00:16.705325 139762286749440 logging_writer.py:48] [36] global_step=36, grad_norm=0.5, loss=0.537294 +I0912 10:00:16.710209 139836003767488 submission.py:307] 36) loss = 0.537, grad_norm = 0.500 +I0912 10:00:16.790243 139762278356736 logging_writer.py:48] [37] global_step=37, grad_norm=0.5, loss=0.522609 +I0912 10:00:16.797201 139836003767488 submission.py:307] 37) loss = 0.523, grad_norm = 0.500 +I0912 10:00:16.876106 139762286749440 logging_writer.py:48] [38] global_step=38, grad_norm=0.5, loss=0.522248 +I0912 10:00:16.879700 139836003767488 submission.py:307] 38) loss = 0.522, grad_norm = 0.500 +I0912 10:00:16.964252 139762278356736 logging_writer.py:48] [39] global_step=39, grad_norm=0.5, loss=0.505654 +I0912 10:00:16.968294 139836003767488 submission.py:307] 39) loss = 0.506, grad_norm = 0.500 +I0912 10:00:17.048798 139762286749440 logging_writer.py:48] [40] global_step=40, grad_norm=0.5, loss=0.486814 +I0912 10:00:17.053475 139836003767488 submission.py:307] 40) loss = 0.487, grad_norm = 0.500 +I0912 10:00:17.137015 139762278356736 logging_writer.py:48] [41] global_step=41, grad_norm=0.499999, loss=0.458191 +I0912 10:00:17.143521 139836003767488 submission.py:307] 41) loss = 0.458, grad_norm = 0.500 +I0912 10:00:17.216113 139762286749440 logging_writer.py:48] [42] global_step=42, grad_norm=0.499999, loss=0.496682 +I0912 10:00:17.220618 139836003767488 submission.py:307] 42) loss = 0.497, grad_norm = 0.500 +I0912 10:00:17.298734 139762278356736 logging_writer.py:48] [43] global_step=43, grad_norm=0.499999, loss=0.464295 +I0912 10:00:17.303659 139836003767488 submission.py:307] 43) loss = 0.464, grad_norm = 0.500 +I0912 10:00:17.376572 139762286749440 logging_writer.py:48] [44] global_step=44, grad_norm=0.499999, loss=0.429691 +I0912 10:00:17.381268 139836003767488 submission.py:307] 44) loss = 0.430, grad_norm = 0.500 +I0912 10:00:17.462923 139762278356736 logging_writer.py:48] [45] global_step=45, grad_norm=0.499999, loss=0.454017 +I0912 10:00:17.467105 139836003767488 submission.py:307] 45) loss = 0.454, grad_norm = 0.500 +I0912 10:00:17.545727 139762286749440 logging_writer.py:48] [46] global_step=46, grad_norm=0.499999, loss=0.421364 +I0912 10:00:17.549635 139836003767488 submission.py:307] 46) loss = 0.421, grad_norm = 0.500 +I0912 10:00:17.628106 139762278356736 logging_writer.py:48] [47] global_step=47, grad_norm=0.499999, loss=0.406272 +I0912 10:00:17.631999 139836003767488 submission.py:307] 47) loss = 0.406, grad_norm = 0.500 +I0912 10:00:17.760321 139762286749440 logging_writer.py:48] [48] global_step=48, grad_norm=0.499999, loss=0.368457 +I0912 10:00:17.764751 139836003767488 submission.py:307] 48) loss = 0.368, grad_norm = 0.500 +I0912 10:00:18.055750 139762278356736 logging_writer.py:48] [49] global_step=49, grad_norm=0.499999, loss=0.369415 +I0912 10:00:18.059242 139836003767488 submission.py:307] 49) loss = 0.369, grad_norm = 0.500 +I0912 10:00:18.282362 139762286749440 logging_writer.py:48] [50] global_step=50, grad_norm=0.499999, loss=0.361316 +I0912 10:00:18.287015 139836003767488 submission.py:307] 50) loss = 0.361, grad_norm = 0.500 +I0912 10:00:18.468464 139762278356736 logging_writer.py:48] [51] global_step=51, grad_norm=0.499999, loss=0.379211 +I0912 10:00:18.472855 139836003767488 submission.py:307] 51) loss = 0.379, grad_norm = 0.500 +I0912 10:00:18.636729 139762286749440 logging_writer.py:48] [52] global_step=52, grad_norm=0.499999, loss=0.459316 +I0912 10:00:18.643425 139836003767488 submission.py:307] 52) loss = 0.459, grad_norm = 0.500 +I0912 10:00:18.840166 139762278356736 logging_writer.py:48] [53] global_step=53, grad_norm=0.499999, loss=0.401407 +I0912 10:00:18.844311 139836003767488 submission.py:307] 53) loss = 0.401, grad_norm = 0.500 +I0912 10:00:18.997939 139762286749440 logging_writer.py:48] [54] global_step=54, grad_norm=0.499999, loss=0.373171 +I0912 10:00:19.002931 139836003767488 submission.py:307] 54) loss = 0.373, grad_norm = 0.500 +I0912 10:00:19.229157 139762278356736 logging_writer.py:48] [55] global_step=55, grad_norm=0.499999, loss=0.366785 +I0912 10:00:19.232882 139836003767488 submission.py:307] 55) loss = 0.367, grad_norm = 0.500 +I0912 10:00:19.413414 139762286749440 logging_writer.py:48] [56] global_step=56, grad_norm=0.499999, loss=0.406584 +I0912 10:00:19.417330 139836003767488 submission.py:307] 56) loss = 0.407, grad_norm = 0.500 +I0912 10:00:19.616340 139762278356736 logging_writer.py:48] [57] global_step=57, grad_norm=0.499999, loss=0.382917 +I0912 10:00:19.622395 139836003767488 submission.py:307] 57) loss = 0.383, grad_norm = 0.500 +I0912 10:00:19.855383 139762286749440 logging_writer.py:48] [58] global_step=58, grad_norm=0.499999, loss=0.371658 +I0912 10:00:19.859813 139836003767488 submission.py:307] 58) loss = 0.372, grad_norm = 0.500 +I0912 10:00:20.012697 139762278356736 logging_writer.py:48] [59] global_step=59, grad_norm=0.493553, loss=0.301014 +I0912 10:00:20.016674 139836003767488 submission.py:307] 59) loss = 0.301, grad_norm = 0.494 +I0912 10:00:20.141456 139762286749440 logging_writer.py:48] [60] global_step=60, grad_norm=0.499999, loss=0.383284 +I0912 10:00:20.146620 139836003767488 submission.py:307] 60) loss = 0.383, grad_norm = 0.500 +I0912 10:00:20.274643 139762278356736 logging_writer.py:48] [61] global_step=61, grad_norm=0.432192, loss=0.369177 +I0912 10:00:20.279206 139836003767488 submission.py:307] 61) loss = 0.369, grad_norm = 0.432 +I0912 10:00:20.349474 139762286749440 logging_writer.py:48] [62] global_step=62, grad_norm=0.343891, loss=0.361557 +I0912 10:00:20.353699 139836003767488 submission.py:307] 62) loss = 0.362, grad_norm = 0.344 +I0912 10:00:20.437381 139762278356736 logging_writer.py:48] [63] global_step=63, grad_norm=0.326291, loss=0.318779 +I0912 10:00:20.442504 139836003767488 submission.py:307] 63) loss = 0.319, grad_norm = 0.326 +I0912 10:00:20.520766 139762286749440 logging_writer.py:48] [64] global_step=64, grad_norm=0.302793, loss=0.366642 +I0912 10:00:20.524601 139836003767488 submission.py:307] 64) loss = 0.367, grad_norm = 0.303 +I0912 10:00:20.603925 139762278356736 logging_writer.py:48] [65] global_step=65, grad_norm=0.388717, loss=0.485211 +I0912 10:00:20.608447 139836003767488 submission.py:307] 65) loss = 0.485, grad_norm = 0.389 +I0912 10:00:20.684454 139762286749440 logging_writer.py:48] [66] global_step=66, grad_norm=0.376723, loss=0.422402 +I0912 10:00:20.689242 139836003767488 submission.py:307] 66) loss = 0.422, grad_norm = 0.377 +I0912 10:00:20.769486 139762278356736 logging_writer.py:48] [67] global_step=67, grad_norm=0.269946, loss=0.364713 +I0912 10:00:20.774184 139836003767488 submission.py:307] 67) loss = 0.365, grad_norm = 0.270 +I0912 10:00:20.855484 139762286749440 logging_writer.py:48] [68] global_step=68, grad_norm=0.230135, loss=0.279238 +I0912 10:00:20.859698 139836003767488 submission.py:307] 68) loss = 0.279, grad_norm = 0.230 +I0912 10:00:20.948415 139762278356736 logging_writer.py:48] [69] global_step=69, grad_norm=0.225747, loss=0.365737 +I0912 10:00:20.952463 139836003767488 submission.py:307] 69) loss = 0.366, grad_norm = 0.226 +I0912 10:00:21.156795 139762286749440 logging_writer.py:48] [70] global_step=70, grad_norm=0.221496, loss=0.339239 +I0912 10:00:21.161454 139836003767488 submission.py:307] 70) loss = 0.339, grad_norm = 0.221 +I0912 10:00:21.277769 139762278356736 logging_writer.py:48] [71] global_step=71, grad_norm=0.208224, loss=0.363923 +I0912 10:00:21.282109 139836003767488 submission.py:307] 71) loss = 0.364, grad_norm = 0.208 +I0912 10:00:21.389406 139762286749440 logging_writer.py:48] [72] global_step=72, grad_norm=0.15355, loss=0.357235 +I0912 10:00:21.395350 139836003767488 submission.py:307] 72) loss = 0.357, grad_norm = 0.154 +I0912 10:00:21.509962 139762278356736 logging_writer.py:48] [73] global_step=73, grad_norm=0.371443, loss=0.396796 +I0912 10:00:21.514196 139836003767488 submission.py:307] 73) loss = 0.397, grad_norm = 0.371 +I0912 10:00:21.597121 139762286749440 logging_writer.py:48] [74] global_step=74, grad_norm=0.133837, loss=0.32685 +I0912 10:00:21.602398 139836003767488 submission.py:307] 74) loss = 0.327, grad_norm = 0.134 +I0912 10:00:21.739296 139762278356736 logging_writer.py:48] [75] global_step=75, grad_norm=0.402522, loss=0.374549 +I0912 10:00:21.743507 139836003767488 submission.py:307] 75) loss = 0.375, grad_norm = 0.403 +I0912 10:00:21.983971 139762286749440 logging_writer.py:48] [76] global_step=76, grad_norm=0.167916, loss=0.411325 +I0912 10:00:21.989299 139836003767488 submission.py:307] 76) loss = 0.411, grad_norm = 0.168 +I0912 10:00:22.150099 139762278356736 logging_writer.py:48] [77] global_step=77, grad_norm=0.368587, loss=0.341491 +I0912 10:00:22.155246 139836003767488 submission.py:307] 77) loss = 0.341, grad_norm = 0.369 +I0912 10:00:22.332469 139762286749440 logging_writer.py:48] [78] global_step=78, grad_norm=0.180026, loss=0.406865 +I0912 10:00:22.336388 139836003767488 submission.py:307] 78) loss = 0.407, grad_norm = 0.180 +I0912 10:00:22.589016 139762278356736 logging_writer.py:48] [79] global_step=79, grad_norm=0.267112, loss=0.395995 +I0912 10:00:22.593190 139836003767488 submission.py:307] 79) loss = 0.396, grad_norm = 0.267 +I0912 10:00:22.770967 139762286749440 logging_writer.py:48] [80] global_step=80, grad_norm=0.171392, loss=0.332941 +I0912 10:00:22.778972 139836003767488 submission.py:307] 80) loss = 0.333, grad_norm = 0.171 +I0912 10:00:22.979011 139762278356736 logging_writer.py:48] [81] global_step=81, grad_norm=0.188834, loss=0.30653 +I0912 10:00:22.982898 139836003767488 submission.py:307] 81) loss = 0.307, grad_norm = 0.189 +I0912 10:00:23.156748 139762286749440 logging_writer.py:48] [82] global_step=82, grad_norm=0.147552, loss=0.367426 +I0912 10:00:23.160904 139836003767488 submission.py:307] 82) loss = 0.367, grad_norm = 0.148 +I0912 10:00:23.337747 139762278356736 logging_writer.py:48] [83] global_step=83, grad_norm=0.160781, loss=0.330308 +I0912 10:00:23.342200 139836003767488 submission.py:307] 83) loss = 0.330, grad_norm = 0.161 +I0912 10:00:23.519801 139762286749440 logging_writer.py:48] [84] global_step=84, grad_norm=0.328476, loss=0.396852 +I0912 10:00:23.524717 139836003767488 submission.py:307] 84) loss = 0.397, grad_norm = 0.328 +I0912 10:00:23.818300 139762278356736 logging_writer.py:48] [85] global_step=85, grad_norm=0.211276, loss=0.325102 +I0912 10:00:23.823282 139836003767488 submission.py:307] 85) loss = 0.325, grad_norm = 0.211 +I0912 10:00:24.158053 139762286749440 logging_writer.py:48] [86] global_step=86, grad_norm=0.110922, loss=0.315182 +I0912 10:00:24.162345 139836003767488 submission.py:307] 86) loss = 0.315, grad_norm = 0.111 +I0912 10:00:24.373368 139762278356736 logging_writer.py:48] [87] global_step=87, grad_norm=0.261542, loss=0.441381 +I0912 10:00:24.377368 139836003767488 submission.py:307] 87) loss = 0.441, grad_norm = 0.262 +I0912 10:00:24.598803 139762286749440 logging_writer.py:48] [88] global_step=88, grad_norm=0.174132, loss=0.350425 +I0912 10:00:24.604117 139836003767488 submission.py:307] 88) loss = 0.350, grad_norm = 0.174 +I0912 10:00:24.706064 139762278356736 logging_writer.py:48] [89] global_step=89, grad_norm=0.15331, loss=0.305158 +I0912 10:00:24.711227 139836003767488 submission.py:307] 89) loss = 0.305, grad_norm = 0.153 +I0912 10:00:24.822026 139762286749440 logging_writer.py:48] [90] global_step=90, grad_norm=0.123949, loss=0.346868 +I0912 10:00:24.827980 139836003767488 submission.py:307] 90) loss = 0.347, grad_norm = 0.124 +I0912 10:00:24.934975 139762278356736 logging_writer.py:48] [91] global_step=91, grad_norm=0.167816, loss=0.385147 +I0912 10:00:24.938880 139836003767488 submission.py:307] 91) loss = 0.385, grad_norm = 0.168 +I0912 10:00:25.074023 139762286749440 logging_writer.py:48] [92] global_step=92, grad_norm=0.196162, loss=0.360914 +I0912 10:00:25.080832 139836003767488 submission.py:307] 92) loss = 0.361, grad_norm = 0.196 +I0912 10:00:25.161903 139762278356736 logging_writer.py:48] [93] global_step=93, grad_norm=0.19375, loss=0.449089 +I0912 10:00:25.167009 139836003767488 submission.py:307] 93) loss = 0.449, grad_norm = 0.194 +I0912 10:00:25.243176 139762286749440 logging_writer.py:48] [94] global_step=94, grad_norm=0.353427, loss=0.301347 +I0912 10:00:25.247222 139836003767488 submission.py:307] 94) loss = 0.301, grad_norm = 0.353 +I0912 10:00:25.334042 139762278356736 logging_writer.py:48] [95] global_step=95, grad_norm=0.499999, loss=0.414595 +I0912 10:00:25.337981 139836003767488 submission.py:307] 95) loss = 0.415, grad_norm = 0.500 +I0912 10:00:25.459582 139762286749440 logging_writer.py:48] [96] global_step=96, grad_norm=0.411088, loss=0.322359 +I0912 10:00:25.464375 139836003767488 submission.py:307] 96) loss = 0.322, grad_norm = 0.411 +I0912 10:00:25.584042 139762278356736 logging_writer.py:48] [97] global_step=97, grad_norm=0.163534, loss=0.291713 +I0912 10:00:25.591790 139836003767488 submission.py:307] 97) loss = 0.292, grad_norm = 0.164 +I0912 10:00:25.673306 139762286749440 logging_writer.py:48] [98] global_step=98, grad_norm=0.202684, loss=0.360248 +I0912 10:00:25.677397 139836003767488 submission.py:307] 98) loss = 0.360, grad_norm = 0.203 +I0912 10:00:25.753309 139762278356736 logging_writer.py:48] [99] global_step=99, grad_norm=0.217855, loss=0.36688 +I0912 10:00:25.757547 139836003767488 submission.py:307] 99) loss = 0.367, grad_norm = 0.218 +I0912 10:00:25.833331 139762286749440 logging_writer.py:48] [100] global_step=100, grad_norm=0.220234, loss=0.251813 +I0912 10:00:25.837366 139836003767488 submission.py:307] 100) loss = 0.252, grad_norm = 0.220 +I0912 10:02:05.015285 139836003767488 spec.py:333] Evaluating on the training split. +I0912 10:02:06.989771 139836003767488 spec.py:346] Evaluating on the validation split. +I0912 10:02:09.628086 139836003767488 spec.py:363] Evaluating on the test split. +I0912 10:02:11.944815 139836003767488 submission_runner.py:516] Time since start: 1145.70s, Step: 268, {'train/ssim': 0.7188784054347447, 'train/loss': 0.2936295100620815, 'validation/ssim': 0.6924304865556415, 'validation/loss': 0.31603948950566263, 'validation/num_examples': 3554, 'test/ssim': 0.7102057598916155, 'test/loss': 0.31816023097162105, 'test/num_examples': 3581, 'score': 387.70237612724304, 'total_duration': 1145.6982672214508, 'accumulated_submission_time': 387.70237612724304, 'accumulated_eval_time': 755.9916512966156, 'accumulated_logging_time': 0.1044626235961914} +I0912 10:02:12.175709 139762278356736 logging_writer.py:48] [268] accumulated_eval_time=755.992, accumulated_logging_time=0.104463, accumulated_submission_time=387.702, global_step=268, preemption_count=0, score=387.702, test/loss=0.31816, test/num_examples=3581, test/ssim=0.710206, total_duration=1145.7, train/loss=0.29363, train/ssim=0.718878, validation/loss=0.316039, validation/num_examples=3554, validation/ssim=0.69243 +I0912 10:04:04.789247 139836003767488 spec.py:333] Evaluating on the training split. +I0912 10:04:06.779339 139836003767488 spec.py:346] Evaluating on the validation split. +I0912 10:04:09.454652 139836003767488 spec.py:363] Evaluating on the test split. +I0912 10:04:11.800642 139836003767488 submission_runner.py:516] Time since start: 1265.55s, Step: 335, {'train/ssim': 0.7157464708600726, 'train/loss': 0.29220608302525114, 'validation/ssim': 0.6924700546479319, 'validation/loss': 0.31335792691773356, 'validation/num_examples': 3554, 'test/ssim': 0.709014031847773, 'test/loss': 0.3156670104588453, 'test/num_examples': 3581, 'score': 499.18704104423523, 'total_duration': 1265.554081916809, 'accumulated_submission_time': 499.18704104423523, 'accumulated_eval_time': 763.003143787384, 'accumulated_logging_time': 0.3435368537902832} +I0912 10:04:11.822579 139762286749440 logging_writer.py:48] [335] accumulated_eval_time=763.003, accumulated_logging_time=0.343537, accumulated_submission_time=499.187, global_step=335, preemption_count=0, score=499.187, test/loss=0.315667, test/num_examples=3581, test/ssim=0.709014, total_duration=1265.55, train/loss=0.292206, train/ssim=0.715746, validation/loss=0.313358, validation/num_examples=3554, validation/ssim=0.69247 +I0912 10:06:03.384371 139836003767488 spec.py:333] Evaluating on the training split. +I0912 10:06:05.366636 139836003767488 spec.py:346] Evaluating on the validation split. +I0912 10:06:08.427825 139836003767488 spec.py:363] Evaluating on the test split. +I0912 10:06:10.746102 139836003767488 submission_runner.py:516] Time since start: 1384.50s, Step: 404, {'train/ssim': 0.7273180144173759, 'train/loss': 0.2857339722769601, 'validation/ssim': 0.7019728540904615, 'validation/loss': 0.3074569916819077, 'validation/num_examples': 3554, 'test/ssim': 0.7191851034539933, 'test/loss': 0.30963214873202316, 'test/num_examples': 3581, 'score': 609.647013425827, 'total_duration': 1384.4995515346527, 'accumulated_submission_time': 609.647013425827, 'accumulated_eval_time': 770.3649923801422, 'accumulated_logging_time': 0.37395644187927246} +I0912 10:06:10.767652 139762278356736 logging_writer.py:48] [404] accumulated_eval_time=770.365, accumulated_logging_time=0.373956, accumulated_submission_time=609.647, global_step=404, preemption_count=0, score=609.647, test/loss=0.309632, test/num_examples=3581, test/ssim=0.719185, total_duration=1384.5, train/loss=0.285734, train/ssim=0.727318, validation/loss=0.307457, validation/num_examples=3554, validation/ssim=0.701973 +I0912 10:08:01.311685 139836003767488 spec.py:333] Evaluating on the training split. +I0912 10:08:03.314731 139836003767488 spec.py:346] Evaluating on the validation split. +I0912 10:08:06.377772 139836003767488 spec.py:363] Evaluating on the test split. +I0912 10:08:08.762213 139836003767488 submission_runner.py:516] Time since start: 1502.52s, Step: 474, {'train/ssim': 0.7291603088378906, 'train/loss': 0.28347143105098177, 'validation/ssim': 0.7030225763444359, 'validation/loss': 0.30528373482124016, 'validation/num_examples': 3554, 'test/ssim': 0.7202537726237433, 'test/loss': 0.30742925853244557, 'test/num_examples': 3581, 'score': 719.0911104679108, 'total_duration': 1502.5156593322754, 'accumulated_submission_time': 719.0911104679108, 'accumulated_eval_time': 777.8156218528748, 'accumulated_logging_time': 0.4036269187927246} +I0912 10:08:08.783093 139762286749440 logging_writer.py:48] [474] accumulated_eval_time=777.816, accumulated_logging_time=0.403627, accumulated_submission_time=719.091, global_step=474, preemption_count=0, score=719.091, test/loss=0.307429, test/num_examples=3581, test/ssim=0.720254, total_duration=1502.52, train/loss=0.283471, train/ssim=0.72916, validation/loss=0.305284, validation/num_examples=3554, validation/ssim=0.703023 +I0912 10:08:49.755251 139762278356736 logging_writer.py:48] [500] global_step=500, grad_norm=0.11164, loss=0.323169 +I0912 10:08:49.758363 139836003767488 submission.py:307] 500) loss = 0.323, grad_norm = 0.112 +I0912 10:09:59.607403 139836003767488 spec.py:333] Evaluating on the training split. +I0912 10:10:01.596095 139836003767488 spec.py:346] Evaluating on the validation split. +I0912 10:10:04.594940 139836003767488 spec.py:363] Evaluating on the test split. +I0912 10:10:07.009026 139836003767488 submission_runner.py:516] Time since start: 1620.76s, Step: 544, {'train/ssim': 0.7331872667585101, 'train/loss': 0.2811053821018764, 'validation/ssim': 0.707059826955543, 'validation/loss': 0.30297429090681277, 'validation/num_examples': 3554, 'test/ssim': 0.7242879220408056, 'test/loss': 0.3050175432185493, 'test/num_examples': 3581, 'score': 828.81680727005, 'total_duration': 1620.7624669075012, 'accumulated_submission_time': 828.81680727005, 'accumulated_eval_time': 785.2173452377319, 'accumulated_logging_time': 0.43327999114990234} +I0912 10:10:07.031507 139762286749440 logging_writer.py:48] [544] accumulated_eval_time=785.217, accumulated_logging_time=0.43328, accumulated_submission_time=828.817, global_step=544, preemption_count=0, score=828.817, test/loss=0.305018, test/num_examples=3581, test/ssim=0.724288, total_duration=1620.76, train/loss=0.281105, train/ssim=0.733187, validation/loss=0.302974, validation/num_examples=3554, validation/ssim=0.70706 +I0912 10:11:58.150328 139836003767488 spec.py:333] Evaluating on the training split. +I0912 10:12:00.140057 139836003767488 spec.py:346] Evaluating on the validation split. +I0912 10:12:03.171502 139836003767488 spec.py:363] Evaluating on the test split. +I0912 10:12:05.496271 139836003767488 submission_runner.py:516] Time since start: 1739.25s, Step: 613, {'train/ssim': 0.7336398533412388, 'train/loss': 0.2801985059465681, 'validation/ssim': 0.7073780890589828, 'validation/loss': 0.301891457853387, 'validation/num_examples': 3554, 'test/ssim': 0.7245739231359956, 'test/loss': 0.3040621836602904, 'test/num_examples': 3581, 'score': 938.8406875133514, 'total_duration': 1739.2497344017029, 'accumulated_submission_time': 938.8406875133514, 'accumulated_eval_time': 792.5633902549744, 'accumulated_logging_time': 0.4639852046966553} +I0912 10:12:05.527805 139762278356736 logging_writer.py:48] [613] accumulated_eval_time=792.563, accumulated_logging_time=0.463985, accumulated_submission_time=938.841, global_step=613, preemption_count=0, score=938.841, test/loss=0.304062, test/num_examples=3581, test/ssim=0.724574, total_duration=1739.25, train/loss=0.280199, train/ssim=0.73364, validation/loss=0.301891, validation/num_examples=3554, validation/ssim=0.707378 +I0912 10:13:56.732705 139836003767488 spec.py:333] Evaluating on the training split. +I0912 10:13:58.711509 139836003767488 spec.py:346] Evaluating on the validation split. +I0912 10:14:01.557891 139836003767488 spec.py:363] Evaluating on the test split. +I0912 10:14:03.865007 139836003767488 submission_runner.py:516] Time since start: 1857.62s, Step: 681, {'train/ssim': 0.7347519738333566, 'train/loss': 0.2789818559374128, 'validation/ssim': 0.7084331007975169, 'validation/loss': 0.30075418432575973, 'validation/num_examples': 3554, 'test/ssim': 0.7255433270865331, 'test/loss': 0.3028992943442823, 'test/num_examples': 3581, 'score': 1048.9334847927094, 'total_duration': 1857.6184387207031, 'accumulated_submission_time': 1048.9334847927094, 'accumulated_eval_time': 799.6957902908325, 'accumulated_logging_time': 0.5035965442657471} +I0912 10:14:03.886895 139762286749440 logging_writer.py:48] [681] accumulated_eval_time=799.696, accumulated_logging_time=0.503597, accumulated_submission_time=1048.93, global_step=681, preemption_count=0, score=1048.93, test/loss=0.302899, test/num_examples=3581, test/ssim=0.725543, total_duration=1857.62, train/loss=0.278982, train/ssim=0.734752, validation/loss=0.300754, validation/num_examples=3554, validation/ssim=0.708433 +I0912 10:15:54.743454 139836003767488 spec.py:333] Evaluating on the training split. +I0912 10:15:56.726143 139836003767488 spec.py:346] Evaluating on the validation split. +I0912 10:15:59.725907 139836003767488 spec.py:363] Evaluating on the test split. +I0912 10:16:02.140598 139836003767488 submission_runner.py:516] Time since start: 1975.89s, Step: 747, {'train/ssim': 0.7340025901794434, 'train/loss': 0.2780978339059012, 'validation/ssim': 0.7084184001521173, 'validation/loss': 0.29969329919852633, 'validation/num_examples': 3554, 'test/ssim': 0.7255832104335381, 'test/loss': 0.30176074409644654, 'test/num_examples': 3581, 'score': 1158.696293592453, 'total_duration': 1975.894044160843, 'accumulated_submission_time': 1158.696293592453, 'accumulated_eval_time': 807.0930454730988, 'accumulated_logging_time': 0.5335135459899902} +I0912 10:16:02.162722 139762278356736 logging_writer.py:48] [747] accumulated_eval_time=807.093, accumulated_logging_time=0.533514, accumulated_submission_time=1158.7, global_step=747, preemption_count=0, score=1158.7, test/loss=0.301761, test/num_examples=3581, test/ssim=0.725583, total_duration=1975.89, train/loss=0.278098, train/ssim=0.734003, validation/loss=0.299693, validation/num_examples=3554, validation/ssim=0.708418 +I0912 10:17:54.124451 139836003767488 spec.py:333] Evaluating on the training split. +I0912 10:17:56.118251 139836003767488 spec.py:346] Evaluating on the validation split. +I0912 10:17:59.142168 139836003767488 spec.py:363] Evaluating on the test split. +I0912 10:18:01.599387 139836003767488 submission_runner.py:516] Time since start: 2095.35s, Step: 820, {'train/ssim': 0.737079279763358, 'train/loss': 0.2770411797932216, 'validation/ssim': 0.7108103462867543, 'validation/loss': 0.29881005831897156, 'validation/num_examples': 3554, 'test/ssim': 0.7277975884005864, 'test/loss': 0.30093764726158895, 'test/num_examples': 3581, 'score': 1269.4936056137085, 'total_duration': 2095.3528411388397, 'accumulated_submission_time': 1269.4936056137085, 'accumulated_eval_time': 814.5684497356415, 'accumulated_logging_time': 0.5636622905731201} +I0912 10:18:01.621841 139762286749440 logging_writer.py:48] [820] accumulated_eval_time=814.568, accumulated_logging_time=0.563662, accumulated_submission_time=1269.49, global_step=820, preemption_count=0, score=1269.49, test/loss=0.300938, test/num_examples=3581, test/ssim=0.727798, total_duration=2095.35, train/loss=0.277041, train/ssim=0.737079, validation/loss=0.29881, validation/num_examples=3554, validation/ssim=0.71081 +I0912 10:19:54.688324 139836003767488 spec.py:333] Evaluating on the training split. +I0912 10:19:56.672692 139836003767488 spec.py:346] Evaluating on the validation split. +I0912 10:20:00.066241 139836003767488 spec.py:363] Evaluating on the test split. +I0912 10:20:02.589267 139836003767488 submission_runner.py:516] Time since start: 2216.34s, Step: 889, {'train/ssim': 0.7376507350376674, 'train/loss': 0.2759171043123518, 'validation/ssim': 0.7114592355224747, 'validation/loss': 0.29776019867578785, 'validation/num_examples': 3554, 'test/ssim': 0.728624434951829, 'test/loss': 0.2997621452631946, 'test/num_examples': 3581, 'score': 1381.459115743637, 'total_duration': 2216.3427169322968, 'accumulated_submission_time': 1381.459115743637, 'accumulated_eval_time': 822.4694921970367, 'accumulated_logging_time': 0.5941040515899658} +I0912 10:20:02.611867 139762278356736 logging_writer.py:48] [889] accumulated_eval_time=822.469, accumulated_logging_time=0.594104, accumulated_submission_time=1381.46, global_step=889, preemption_count=0, score=1381.46, test/loss=0.299762, test/num_examples=3581, test/ssim=0.728624, total_duration=2216.34, train/loss=0.275917, train/ssim=0.737651, validation/loss=0.29776, validation/num_examples=3554, validation/ssim=0.711459 +I0912 10:21:53.179136 139836003767488 spec.py:333] Evaluating on the training split. +I0912 10:21:55.164557 139836003767488 spec.py:346] Evaluating on the validation split. +I0912 10:21:58.383379 139836003767488 spec.py:363] Evaluating on the test split. +I0912 10:22:00.579999 139836003767488 submission_runner.py:516] Time since start: 2334.33s, Step: 961, {'train/ssim': 0.7378678321838379, 'train/loss': 0.2757465158190046, 'validation/ssim': 0.7114406192846089, 'validation/loss': 0.2976615875707829, 'validation/num_examples': 3554, 'test/ssim': 0.7284563113044541, 'test/loss': 0.2996555851411966, 'test/num_examples': 3581, 'score': 1490.9036588668823, 'total_duration': 2334.33344912529, 'accumulated_submission_time': 1490.9036588668823, 'accumulated_eval_time': 829.8707458972931, 'accumulated_logging_time': 0.6444931030273438} +I0912 10:22:00.601509 139762286749440 logging_writer.py:48] [961] accumulated_eval_time=829.871, accumulated_logging_time=0.644493, accumulated_submission_time=1490.9, global_step=961, preemption_count=0, score=1490.9, test/loss=0.299656, test/num_examples=3581, test/ssim=0.728456, total_duration=2334.33, train/loss=0.275747, train/ssim=0.737868, validation/loss=0.297662, validation/num_examples=3554, validation/ssim=0.711441 +I0912 10:22:09.027065 139762278356736 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.131386, loss=0.352369 +I0912 10:22:09.030334 139836003767488 submission.py:307] 1000) loss = 0.352, grad_norm = 0.131 +I0912 10:22:42.398454 139762286749440 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.0762359, loss=0.2711 +I0912 10:22:42.401585 139836003767488 submission.py:307] 1500) loss = 0.271, grad_norm = 0.076 +I0912 10:23:06.388797 139762278356736 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.0484939, loss=0.364141 +I0912 10:23:06.391752 139836003767488 submission.py:307] 2000) loss = 0.364, grad_norm = 0.048 +I0912 10:23:30.293541 139762286749440 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.0825935, loss=0.303391 +I0912 10:23:30.296624 139836003767488 submission.py:307] 2500) loss = 0.303, grad_norm = 0.083 +I0912 10:23:51.141583 139836003767488 spec.py:333] Evaluating on the training split. +I0912 10:23:53.084968 139836003767488 spec.py:346] Evaluating on the validation split. +I0912 10:23:56.795864 139836003767488 spec.py:363] Evaluating on the test split. +I0912 10:23:58.919345 139836003767488 submission_runner.py:516] Time since start: 2452.67s, Step: 2927, {'train/ssim': 0.7458196367536273, 'train/loss': 0.2680674280439104, 'validation/ssim': 0.7191054258845667, 'validation/loss': 0.29051456655353125, 'validation/num_examples': 3554, 'test/ssim': 0.736318784797368, 'test/loss': 0.29202163981778834, 'test/num_examples': 3581, 'score': 1599.721081495285, 'total_duration': 2452.672803878784, 'accumulated_submission_time': 1599.721081495285, 'accumulated_eval_time': 837.6486473083496, 'accumulated_logging_time': 0.6740052700042725} +I0912 10:23:58.940202 139762278356736 logging_writer.py:48] [2927] accumulated_eval_time=837.649, accumulated_logging_time=0.674005, accumulated_submission_time=1599.72, global_step=2927, preemption_count=0, score=1599.72, test/loss=0.292022, test/num_examples=3581, test/ssim=0.736319, total_duration=2452.67, train/loss=0.268067, train/ssim=0.74582, validation/loss=0.290515, validation/num_examples=3554, validation/ssim=0.719105 +I0912 10:24:03.074253 139762286749440 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.0923818, loss=0.324035 +I0912 10:24:03.077343 139836003767488 submission.py:307] 3000) loss = 0.324, grad_norm = 0.092 +I0912 10:24:27.005277 139762278356736 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.179975, loss=0.299216 +I0912 10:24:27.008391 139836003767488 submission.py:307] 3500) loss = 0.299, grad_norm = 0.180 +I0912 10:24:50.897279 139762286749440 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.201003, loss=0.288526 +I0912 10:24:50.900299 139836003767488 submission.py:307] 4000) loss = 0.289, grad_norm = 0.201 +I0912 10:25:14.780320 139762278356736 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.188659, loss=0.251032 +I0912 10:25:14.783301 139836003767488 submission.py:307] 4500) loss = 0.251, grad_norm = 0.189 +I0912 10:25:38.681326 139762286749440 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.0370094, loss=0.241917 +I0912 10:25:38.684377 139836003767488 submission.py:307] 5000) loss = 0.242, grad_norm = 0.037 +I0912 10:25:49.454987 139836003767488 spec.py:333] Evaluating on the training split. +I0912 10:25:51.396963 139836003767488 spec.py:346] Evaluating on the validation split. +I0912 10:25:54.794511 139836003767488 spec.py:363] Evaluating on the test split. +I0912 10:25:56.864700 139836003767488 submission_runner.py:516] Time since start: 2570.62s, Step: 5216, {'train/ssim': 0.749497549874442, 'train/loss': 0.26447248458862305, 'validation/ssim': 0.7226716375914463, 'validation/loss': 0.28728914878042344, 'validation/num_examples': 3554, 'test/ssim': 0.7397906812866518, 'test/loss': 0.2887677382038013, 'test/num_examples': 3581, 'score': 1708.4362881183624, 'total_duration': 2570.618153810501, 'accumulated_submission_time': 1708.4362881183624, 'accumulated_eval_time': 845.0584487915039, 'accumulated_logging_time': 0.7028579711914062} +I0912 10:25:56.885050 139762278356736 logging_writer.py:48] [5216] accumulated_eval_time=845.058, accumulated_logging_time=0.702858, accumulated_submission_time=1708.44, global_step=5216, preemption_count=0, score=1708.44, test/loss=0.288768, test/num_examples=3581, test/ssim=0.739791, total_duration=2570.62, train/loss=0.264472, train/ssim=0.749498, validation/loss=0.287289, validation/num_examples=3554, validation/ssim=0.722672 +I0912 10:26:11.068797 139762286749440 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.0679424, loss=0.274346 +I0912 10:26:11.071854 139836003767488 submission.py:307] 5500) loss = 0.274, grad_norm = 0.068 +I0912 10:26:35.028560 139762278356736 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.0931679, loss=0.375896 +I0912 10:26:35.031643 139836003767488 submission.py:307] 6000) loss = 0.376, grad_norm = 0.093 +I0912 10:26:58.859849 139762286749440 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.0987513, loss=0.271 +I0912 10:26:58.862875 139836003767488 submission.py:307] 6500) loss = 0.271, grad_norm = 0.099 +I0912 10:27:22.710476 139762278356736 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.032298, loss=0.308556 +I0912 10:27:22.713657 139836003767488 submission.py:307] 7000) loss = 0.309, grad_norm = 0.032 +I0912 10:27:46.611001 139762286749440 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.0426679, loss=0.222867 +I0912 10:27:46.614065 139836003767488 submission.py:307] 7500) loss = 0.223, grad_norm = 0.043 +I0912 10:27:47.439240 139836003767488 spec.py:333] Evaluating on the training split. +I0912 10:27:49.397877 139836003767488 spec.py:346] Evaluating on the validation split. +I0912 10:27:51.382219 139836003767488 spec.py:363] Evaluating on the test split. +I0912 10:27:53.360685 139836003767488 submission_runner.py:516] Time since start: 2687.11s, Step: 7507, {'train/ssim': 0.751577513558524, 'train/loss': 0.26305365562438965, 'validation/ssim': 0.7241025462067389, 'validation/loss': 0.2862729666671796, 'validation/num_examples': 3554, 'test/ssim': 0.7413039986430118, 'test/loss': 0.2875992243132156, 'test/num_examples': 3581, 'score': 1817.1823732852936, 'total_duration': 2687.114144563675, 'accumulated_submission_time': 1817.1823732852936, 'accumulated_eval_time': 850.9800071716309, 'accumulated_logging_time': 0.7312157154083252} +I0912 10:27:53.380898 139762278356736 logging_writer.py:48] [7507] accumulated_eval_time=850.98, accumulated_logging_time=0.731216, accumulated_submission_time=1817.18, global_step=7507, preemption_count=0, score=1817.18, test/loss=0.287599, test/num_examples=3581, test/ssim=0.741304, total_duration=2687.11, train/loss=0.263054, train/ssim=0.751578, validation/loss=0.286273, validation/num_examples=3554, validation/ssim=0.724103 +I0912 10:27:53.929469 139762286749440 logging_writer.py:48] [7507] global_step=7507, preemption_count=0, score=1817.18 +I0912 10:27:54.085808 139836003767488 submission_runner.py:857] Final fastmri score: 1817.1823732852936 diff --git a/logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..566cba6e --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,16 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/loss,test/num_examples,test/ssim,total_duration,train/loss,train/ssim,validation/loss,validation/num_examples,validation/ssim +749.0619850158691,0.0,276.97007966041565,1,0,276.97007966041565,0.976249432770176,3581,0.2693324979874249,1026.6488287448883,0.9745864868164062,0.2550915649959019,0.9774193278040588,3554,0.2467691210058209 +755.9916512966156,0.1044626235961914,387.70237612724304,268,0,387.70237612724304,0.318160230971621,3581,0.7102057598916155,1145.6982672214508,0.2936295100620815,0.7188784054347447,0.3160394895056626,3554,0.6924304865556415 +763.003143787384,0.3435368537902832,499.1870410442352,335,0,499.1870410442352,0.3156670104588453,3581,0.709014031847773,1265.554081916809,0.2922060830252511,0.7157464708600726,0.3133579269177335,3554,0.6924700546479319 +770.3649923801422,0.3739564418792724,609.647013425827,404,0,609.647013425827,0.3096321487320231,3581,0.7191851034539933,1384.4995515346527,0.2857339722769601,0.7273180144173759,0.3074569916819077,3554,0.7019728540904615 +777.8156218528748,0.4036269187927246,719.0911104679108,474,0,719.0911104679108,0.3074292585324455,3581,0.7202537726237433,1502.5156593322754,0.2834714310509817,0.7291603088378906,0.3052837348212401,3554,0.7030225763444359 +785.2173452377319,0.4332799911499023,828.81680727005,544,0,828.81680727005,0.3050175432185493,3581,0.7242879220408056,1620.7624669075012,0.2811053821018764,0.7331872667585101,0.3029742909068127,3554,0.707059826955543 +792.5633902549744,0.4639852046966553,938.8406875133514,613,0,938.8406875133514,0.3040621836602904,3581,0.7245739231359956,1739.2497344017029,0.2801985059465681,0.7336398533412388,0.301891457853387,3554,0.7073780890589828 +799.6957902908325,0.5035965442657471,1048.9334847927094,681,0,1048.9334847927094,0.3028992943442823,3581,0.7255433270865331,1857.6184387207031,0.2789818559374128,0.7347519738333566,0.3007541843257597,3554,0.7084331007975169 +807.0930454730988,0.5335135459899902,1158.696293592453,747,0,1158.696293592453,0.3017607440964465,3581,0.7255832104335381,1975.894044160843,0.2780978339059012,0.7340025901794434,0.2996932991985263,3554,0.7084184001521173 +814.5684497356415,0.5636622905731201,1269.4936056137085,820,0,1269.4936056137085,0.3009376472615889,3581,0.7277975884005864,2095.35284113884,0.2770411797932216,0.737079279763358,0.2988100583189715,3554,0.7108103462867543 +822.4694921970367,0.5941040515899658,1381.459115743637,889,0,1381.459115743637,0.2997621452631946,3581,0.728624434951829,2216.3427169322968,0.2759171043123518,0.7376507350376674,0.2977601986757878,3554,0.7114592355224747 +829.8707458972931,0.6444931030273438,1490.9036588668823,961,0,1490.9036588668823,0.2996555851411966,3581,0.7284563113044541,2334.33344912529,0.2757465158190046,0.7378678321838379,0.2976615875707829,3554,0.7114406192846089 +837.6486473083496,0.6740052700042725,1599.721081495285,2927,0,1599.721081495285,0.2920216398177883,3581,0.736318784797368,2452.672803878784,0.2680674280439104,0.7458196367536273,0.2905145665535312,3554,0.7191054258845667 +845.0584487915039,0.7028579711914062,1708.4362881183624,5216,0,1708.4362881183624,0.2887677382038013,3581,0.7397906812866518,2570.618153810501,0.264472484588623,0.749497549874442,0.2872891487804234,3554,0.7226716375914463 +850.9800071716309,0.7312157154083252,1817.1823732852936,7507,0,1817.1823732852936,0.2875992243132156,3581,0.7413039986430118,2687.114144563675,0.26305365562438965,0.751577513558524,0.2862729666671796,3554,0.7241025462067389 diff --git a/logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/trial_1/events.out.tfevents.1789206186.5ce0f4169081.38.0.v2 b/logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/trial_1/events.out.tfevents.1789206186.5ce0f4169081.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..8f765dddcd2e82f363364669da4788d85db65489 GIT binary patch literal 39944 zcmbW=cU({F0|)SQt-e-vB&Cd0(vlDwqJg3)G*yUF8WOomqHG!>BV~oM6Owdop_>sI z*~%qPM+ucIV0cPf8FQvdfw+b=X0LE<9vrnG5_5wFnT2;B^9=JRBDlN*H%;g z-MnUNn$7i`9XMELd{2>bpy%uWU;k;|zQLOAzW$z?L4jW0nx6mtdw@%@zgM8Azwe;( ztr~S{73q0qGT(+?`|w|yS`U$;sIj+iK!Ay;d82kx%8eb}=M1-OA(9a_3vhSycJmJ~ zfA#NT?YgmBA|gs<3@qx#_7?RN$xQcm^KhB%>+ka`;9E`2FbL?=be5EET|mp-N)u&T z9Ip#d74;UinC|U6)y>-_AkfqASHyq+GenrUn=RI>i*VNM>?q^7Zuma)|Ic5xky;;7 zFOewlzhlxrAi#@E!zpnwPD9q}cx%YWOXSNCKNewshv2cep>k-{pt^aiXg081#xR9s zXo%EAZRWapdwIA8dimmHS{w?+$(+vKu@QpK?iCCYpHUx!`8cBm+UNf@uA7lmYVJN6 zxg#VOg9jmL#en**@%OpZmq#edIb5`F8i;Xf9%3WK+g5&VNPasId5LhI4Cpl5y>$S>5Jsu+}(qGg1p@VJw03k zf~NX-{r_zCo$V6nWH*N9mPZc|L@)qS7|x~=lE^D zO@T>WT*#-{du4jQBXI*nx*|F8IB(zS)4gU-hs6KJf_maOH>Ck}iFeLBy_DHfK@x?p z{xCep*Z12xWN=+bySUd+Ws1L%kbWX{k*uG;r>BqKe^;H0yKm6!z+a2Nna(Jkdvi4F zlFnbXua+^=_Wso)oF?pWXfDpf>gc^xIIqtQBcmnk!pO%b8o}`h7)2r^J|1gqbf$o! zyPZFykmmNeO-d*#F3V1I1;s~KGFn2#9hH=spirMhB7_xEMsR8bj8N$Zio2VG8O2{* zMTZCBsObFckuoSMQ^{xv6{6C^@dVT6h{KJHgQF`+V)Q8eZ%L?0HQqQi$s zbx@q_xbW9#2^A*Q`yxQ0uS_C@6=A68=y9++SFzE+2Ngy0#+?Sm%wjTH zLPdmW%e$ax|A9mZE0R$0L$T2#P=o|VF^Z;K#oJ3>s8H^@eF7*-7L(BuDoV_q(?Aik zl0*nAa!_%0!kZqT$cY}fe%cI*9A^?CtdKH>jv9Yfb}cAU zls7R78LpyYn;)*lvF|sMK`~p4jFwPw{r-jqy(5UPu3 zyMdy78yPL3LQ8hQ5l|dBNFsz4TBr!GjLCqtc+qPgqiD%hY@HsAYoTW{5oGjCghqGvf7&9C^*KBLl> ztr!i8fl`NlO+Z*-g9_>L(I-JMc2ouv(weKd9mBfo9USLn1d2vBWHi5m^Oy0g;a%2( z;)n~05LURN;_kTdB2Y+m&0-X7xQe%GthI3d;JgYH8&%0@2^CJaJ??`dLWe{MEBsLL z&xsX2ph&Ym%P3^IijAYE;dOC#ce`VtIOa`8^D8(VH9TvpDJY)KB@x1kFjN=^<<13# zbHxQl(Uz+ie%T%sXJ*{RyPn+-GMZn(IivPX%~=8p>6RCNO+Z)?g^How+qXqUWF8aJ zj;nBeV26tN3Dc&+TI||FM)NB;S9j5xd7VJfGMz*SE7qbSzKcm0P&fn@FpBnE#md){ zP;o1wx+y5;tRtf(RO~o8D-IM*x048AMFJ|;?ktrDg`DwSM$v(**cWYz3hg}u6+m&* zij0;}@%_D1E-17-NQAH=2^IMPAMxrQdh#Ko=*U%k)MY7VS&d!Mk85Qw`q#WJy3kUM@I82I9Ipz`fL+Wn7kzs!ir2( zC^WfN4GQCy?-_+0S8+$q7}w&R)PyWhxOE|;`4yZq>bjMBC@2yQNQAH=2Ni>de_RZT z_18WziY{Em1%*+lh>%%y7Zg+PlhOPN&KVW$ILa9mlRuLPVZ|j>th~DzU&+Z&sALpf zxeCkus;Kzh|Hf)i+}TY=^D8)K)R{idn}A}#5fUM+$VbJtkQum-l0N=}QOI)@x59N$ z@y@<@8YoI~$Y_4W5B8pR{2_llqgoV@2w}xtRHTh_8Ukn3RrA00*S=HIjjIr~>yHXe z<@<4<@UkbPB~-jV84v_((bJ7Y2rC|=;_IP4S3&V8ra7Zf;40QsDWF2#+rkkPxyfWS zzk+jhublf5cjo;LkqBW$2`aYbUETtUA?I5$itb#+>4zOrG4G|;VNiJ7Afx#eoa@5% z!N+P?3-yO2LRe9TimCH|oB@UN$99ZDk*f&ZBaey&_a6KNg^pDFU!(aIoR0c?neSmx z7`G%5!iq{%7=3HM92Aiua?CuGxQani*RZxEl=Jq`)Y8a20MHnxkUY)Ycb4v11n*&9C6B#o4P}ia_x;mqZ9F8l&P! zM#LIei?c738HF-eaoR=(73aqV;$KOsAINBa1?M#oSJ(Ivps@QvB7_yqQE|EW$~I86 zl2c_ADqMxvz}Bc(K2Y%~ti?|?GMZme$-ZCo%xY%`injVBLRirf73saJ2ZLg_aX&`U zldI^qlC>85MOJv%OC3!{^DDlx6-h_r@<9>hK_Y||vZ$!Eb8P|&>w`Lsq8C?j{Bjdq ziz_D9Ghr>3Um>F~Eh7=aiVmo#F@AOc6c^PD7)5WcqWNK#;_vm22S9OM zkBsJ5d}Oah$LW3!uog!~kO*Og94hiPS}y@b*sY!RA3LvdBLNYcZz> zi4a!wL`9i($#ziWm5pKMp~h7xjQNR*HY>0F4T@GZWVD2e(<9CCFV?K)V}DIRSfPpv z`xWv7K+#d(mI+bkD!OLALq)`vN%+}=B3m+=U-6c`7EKmR><4Qx$elz8D>P7%lAEss z3V&?}Mxnt~cu74##l7ff`09bIAsNlDc*9nFi@N^-6zP^ELRg`NilO~W?Ld)~x1G#2Qr#pQNmV?@O!c!6sx$Vok0eqtN0irtZo@h04r$8&HH_C8H%& z?2EoI3l#lINrbS%2o;5wA4Py7CN+XlXmb@MDyLCVb+y+QP~>Hj(fkU|8Fg{}l@_3= zJWC>k6(dj)x49-46anh%8HEm4G2>?@Dq0on!1q|)29VJbDjpZNxc~}tYZ4)>Fh|AQ zUei8n$VAF18@fsE!?aMogK{3r`h9F@xWH34CT9V)yEci`XMld8@zA^Kc} zx$GHK9F82E4{Kp6o%3rnzk+jhe|VrU8x)C(Btlr>fC>$jHs!DuWx3axc?{$#Os}3o z#oCORNKkm^kHwqn5CaC}EBH=9HVD?Cu)+@;VP6a|mJFp9xk#TScgR5a_7T?%WVRZd3pE1s|w zTR*-lfwk!VjYJ45W}sq;XU$Sj?0zkkR{P5T5Uyf=_f4n>{?I-V6n(16Xnw_y!{0HY z@bkaZL1ERn(XR;zE4)!Ld0+c9P&`nRVL}YKioBt$Gs;2ayc-mOCS)|f;vsu2lvFa; zgJO>@i4a!!p~ArP`esn*r?+JkL%E96jamCBTZ4l~L6MwEM)NDim_Nj|$X2_JuhYg~ zBN4)iKvWbIE9HS=-GWYxVi;HPN#O)e!RY=8eAoU&G#SmW_`BUfjPSTW`vk1TtppMw zteA(26O$TOfI_!xH%2j>tEkdX!6{tx=-eCD;-M-T&96vkvl}D6imF3E(a)Gf2rEKS zp=^<+1qvI#9*n|>t5}}Ny6dgpE%h7}mqN&Be#P!)iKwWm{^1RZf@l&UtO!FzM3LQj zP$cE_W)wW%L`(9i#3|&R_vizP{C~-4e#M(67cpYP6ZgMBVfTbY2rCw&VqWm8UZ6N$ zrp_o#xN9M2b{D4*dTCPQPjWD#)!BIb^rK=sjb9TGRxCk9Qjq#CP^|RS zWk$PSMkW0 zb#+(#8+r;9)9#be{E883(@`O#W6>8Bw@XNbuwpqX(yA;@gW|Q4C8HS0RpcE^!zpC0 zFL(rs>1t#&zvA|dsTgs2?V|&rSZqKdgcU1M;eN8<;ZlHFAgqOR zJ2IMIVPU9%5w^Eip9e+DVI)FWu@)7HmTkj95qsK|QCM&luT-z%6gr&TCI^ard1N%d zBC{WBb8j|R(-;&1_eg}WVm&G@zDX$ng^kTLMq$ZSJau4g9|C;Cb3ie0HW|&YNNp*D z2f{Zh@)aoJgGq$2A{G_K8nE5>!cj*94E zEhd6uToj2AR>Y&iXp<4X!g;veB1SQqt1#GDfKwPSMl~E1b_!%PzoOemb&PPFF{3Rg zJcf`6VMPKeu3Q+s6V~Fvsc1$qhO2l`n2uA>{Q1-b6bH|f(fkVk3d-27Es)nNk;Q4_B$Bhfv9%2R|Z8$5Qz|0 zY(~YoS=;e1)(&|)8HF`hasSK}oI*{mN?TCq6_L^Wig&qD7~$wCkDnXt_?$!tE4HG- zct<+!%r6HVViY!9#jxPLIEAuvWxAk{*+WM2E2f6`$A}GYPU1J+hFl^M!ipqRyf3N7 z&(q4}WHX9!Tt!6rW}JetR6@|m>Pr-H3 zTa zg)LX1*gX}e;AtL!p9+q=PDb-9ic@-G#GRk}Pr_Ol-X#&jie0FvY_|#bQF(4p7=;~I zp`pdvfvhSC#n%QMgUD!p#b26xP_eq>Qv*<_FC`Jeian_4zRD9nH~2NUf>BK1DzXE1 z;S?rZ2pSA$)QzQNG`}Ln!2u)Gl}F$!IjiDHgs@^SDjc^@yaH#G>rhd8?JJxUxr#`a z?Kp*YO(rOU;>k!dnqM(i{n=3X_~rbsgWJJcjG90qgcbWyQQU9t2~fy3kz*A0Tt)Kt zC#dLVsM!G&@7j>j{EBNOkr*Lwq`wOkLlj7aup%85rdsa!+ThWE-i*S5t08h6{TRh0t|D_N>wXcU*2x0a zqNP0<&94~$qZ=xoywTbV3WXpNA*?uzif7FpFM=~FG((S3Oy(-G6As}N)_xj?uMPSa zlhOQ&XLo<#b&;+or3i{MpGbtT;wUN-{5#_7Q4>E5VH8uiiqG2TQBicq7Qe4&_mhm~ zS9~jE?ac2)j$8!_>*j{PCLpZHK*hhZhSy*%9G;C}LL9k@?8}*`aM6$%31?JkIT_8b z*!RdBBmAS3Qb6%p*6h~=gcX^nIND#Q4=9!gTQVU|Tt({-FK`NXRYLKVoEGt9G{3@p z+DMFWk6GOc6c@LX2w}xZRE(Ry9G`pOnQF}_oVf~ROV&GL%QmcB4r`%(l#J$A_*Wc7 zMZBw-0w@Y{NrbQ>3l#^HdfCHTXwRR(C|tOT2WGc$3OQPiDxjDjO-A!89<$#OYusMC z5)>U`NQAH=8x^-Sit$~0gQZT4!j-ES+;S%>PA}+m0Tl6ZWHi5`?|jyu?mdDN@H79d z_K^r-MGh*ix6f`2YcVg>lTo;F6+u(tP+_B7;|GfTBV;tcVp>oij5rY&gRg-c%Ow%Q zidiWNGu7{yesBGjMNQPy5%_!q0bAsNlDuwQA73VYL#2XIEkS&#@}#W_^O z_PON)iY-=tjKZC($S7pJW}eohOas;;(2`d{rm0YA%8?aCV>{@}pGiSfuuDzqI7&x$jeK||_$M5B_a zHcltcHCq;vHv0-EiO=c%4c1>CTBUPW!(;Vc&ZS9z^pHlUR&8zIS(csurirx+&v{B? zi~IrTTku2H9x_Ccd>P`hmoJj>XisuWA8J_lJiSZDQ^w2KtfUzYd`{!uWaAxLby;zr z^Ic?uVrUlkL5;8Ooy7+=&eJv<1dr6*yNQmk=ShtY`fKnhh)KCizKpD!i2r~SJ7v~x zr$LfFs*y9jM74oYMs=yGoh#yG?xfPx-#@D{xW*7?8L*>!hG|{ip~zlIGJhSSc@jOW zF|6!1ZsW(Ux-(n9F4Ee5_&nphdPA-JD56P!R4ej9nd_kvUOl0S{I&8ioK-7t-#A@_jISkp z8R9beZLGp9(>;|BwesZy&1hh)+#Zy#2DS3z3(exzO8JH=Je}>1n94(~T>nAG*HbH} zw722~d*iL)52%$6jX#raUHZx6Zb-`wAP?04Xc&ay0J3Q2-^$~HMaQgY0|NO z3GG%QvDXDbiHpi)Jn6z>G*nT<$Ien z3{p%lv4mQ=xP%5tS}UCCO?sdC5^AOKDw_KHT5;T9gtPSdc%~F;rDZJ5lc-hehrD^gdmRu))nPJvpvnM_j}Rx6F-UGeY1IR-k)wY5?tSSv<*S+&y3 zX;^Ehm3hXzA>t>xy2?5{(dTb8FNHGLVnIXds};*CR;?sI3cdkJsW!q%;&bXfmi1!! z(|fKJP%AwgXwo0miuKqwgP;oAXJKzkzHjfUk zy-GO0;4wMnIsRez@LA6qI3~{FG(-59RO2z3ygOzMRNGgD+~Kv5q{c$quD*wd;66t; ztE4-Gtm?xXB7ViTc((@ov2W^FCup_Dnlz-oek^Xp`fI1ZimNmv>FNt7iOVC7lO4?aD!DVd$gwGGsbjEc|*ic zbj`V}TDiK+`2>`~tTGx>U#*<##Hy7-D^Dz_?61N};&WQEj`c34a?Is#P%Fc0Xwo0m zN^ab;f1nZ;Hl0e2H-D|9Eo9ZodF|IJkTJOxUxv6$l+G$lgLCGUP%Eh&Xhs8TWsRIq zf2fs2d78zomFvf5;^}<9aPKy#m4B7#_V!}^la zt-h&w3f&bOyQt<3&Llm4hyDkHwA zK_%>$t|G^qzg7mEX4Oi8X7Xgnh?L>W5SMN3$f}i+*|Wz(t^6ZPGa6VcF|7|qL9N7g zrdixtDOQNU)7eG)nHD1tG=Dc|*ic^unpET6y9jg{j5}#A@j|8049v7pVP%8@3Gsw~Tqgr_$&?Ffu;kFD-P`hunw$R@#&Bs2({8lj%GBlR+^{=j)PiJP^4MhTFE=WdV~8?x%?EUl`p;M z_*|fa6$Q2OXb8=d zs8%#avmP_vbL1L(sFf+EG*Yluq{iUH-dC#if?83uqA3ll6`Qt?T%ZxOjd$Owt(7Xl zT1mgds+EnJkMU_0X;!=;;wQSa8LL)e1|K{JWuR+IL+Yy)ZDoDj63R3`IvkR$CkrQu zr3t5$Y6p0b2mNq3`3e^e_=H#{8zm9Qdp4>{iawW7R?RV&+*Chvj_ze9W(;xeE4 ztbLf?@ZJldR!TEzMgwcb@%ah|s1>6en#HY^L+ShRbS~{&)eLGy;UXPhPpyPTim+B5 zhqmYryOogZG)U4~;Y_cIwEP69m4Mqc_4l>Xax3fVT{`D>7}Sc>Lz*X1t>|@@!!!Q1 zliNqAl^!o>q+qQWIpaDgYWuW>TG4(>QyNw)ZKp)yHwxCr_x@O0D`%St?N&;jvuefj z?us_hO=jD8L&Q(?U7D;~NiV%L4ay*C7Y(VeR;rX(wPHFv%>j~jrwJ#C&#B9FR;?tJ zTb_qnu{=hT{-{-L@h^?AWZy(MCOux!48da}H6M?Oe22aF0Wxya#_zR|2Ry%ZyXzY3!CD8D zy4wO|ObQ*&4~grs8Q=2PKddzxAP)Ozh9D%2^(m6C_9h5vRh=Mh`QIlk^Lc)pG{f&C z9?f&{(Q5%xB)yRg=ZCD^J`y1T?=NQpbvz?b!L5t>w)iR14z0z zML0=(PPW;xIH!Ky-?W5V?__tH^hdYe4)0!T!pcvZNfT>Vp7Tu|_eHFo&iTMamm$M+ z4qt}2>_G(U3k-pSPac6=@78%Vqk*?x<4&qu;MTij5zXS>dfR`!h!@j$`%X>a*4uR% z9beC__nc)HJefV7CQg7`@0?XMNYYy`XL^w%J6pr8cXb?1{r#;s!fPCI<;HZYf?Myr z%`{Jwr=uGWQWcYsU@p~Lwh?p6j^LVa)Llma9woMs3@awa3B_vev8 z04a$)(XAGe#Php-drrjQBo-FMbpc4?YC4=Ba_#0A%-FlVM*%?c;%SB;L~1@lc9jp6 z1Bj)}VdYv#EYAqFwQDtT2tJCHQ>EqVv2V;&-*O8O!jPoYEco67KxNuF;VC z_7KvISnu(*&(!M&Nn>vbCyCE#ZD-b%v+wbDf7nATx=)k-Xb;g(J!}$Oi$zaqV(rRv z_7HEU4#mU0I4ffgWMq}{Wr)k1?y+Pxio<@w9wND%W;AdQk>>092=)-mDrpvX5AkKy zZM>-}zHuH4dx)eOI=-Gg#Pd`oJek|xJ(>u6h~`ZPk&{`{Jp^ZZTaNYDg*}9EYnuA| zJw&#tEpk_;yI8^=LboH$lV}eS_MY|Cjx%ZwI?x3NyU|F&E|5yW!{iqmrPOG+7zVrOfThHK?*aZ$}?WN%iA017H^Y^1uuNByj8EMiAa7=91 z(G0<3@^~K}lg2qsz5*ov-C6xwNEy!$b9n@@p6%Btr;<`7 ztjJDnnjr{Lx{8oRqe8j?L?I^4q!x0Q=cjL)ePeyY=i=gsB7h9tOo#JBo>?|V$knuC z;{f8cgJuXqa*7bLMlUA>Ae(h}o7Y0}cz&zIV4w$HBZF6Z;upLf4C!!w$f|Ri2zg~) zcn2Wu&1i-o#Pu~ombV%5171R~o#$X#3%NO1Xb&-Ux)I*gewKvbZ=Pv{^M;7u)hg4M z;xP#*ZPX6#{G+32NPT;V%g(Gl#C5gvfsmB4MmR})PA=G+DH4XG_iK&IeUmM7g&3U4GSNO+eWG3t(uI;B84ctTA zd38Dx_7LZe&@ApA;!dw;cvB5AZqo<$5a}oB_=b+T(NK~>>+IPX`V!Thzc#%2N6H#cU%u$kWok@1-l@D z)dkPgZI8ho;_V}v(y%?my~o#j!6}ilZULzk2~G){v(Y;Ml)s6Nml*Uxu3IU9Vv}h~ L!$5PjjxYZo6gyMv literal 0 HcmV?d00001 diff --git a/logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/trial_1/flags_0.json b/logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/trial_1/flags_0.json new file mode 100644 index 00000000..a6c808c7 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/trial_1/flags_0.json @@ -0,0 +1,62 @@ +{ + "logtostderr": false, + "alsologtostderr": false, + "log_dir": "", + "v": 0, + "verbosity": 0, + "logger_levels": {}, + "stderrthreshold": "fatal", + "showprefixforinfo": true, + "run_with_pdb": false, + "pdb_post_mortem": false, + "pdb": false, + "run_with_profiling": false, + "profile_file": null, + "use_cprofile_for_profiling": true, + "only_check_args": false, + "op_conversion_fallback_to_while_loop": true, + "delta_threshold": 0.5, + "tt_check_filter": false, + "tt_single_core_summaries": false, + "runtime_oom_exit": true, + "hbm_oom_exit": true, + "test_srcdir": "", + "test_tmpdir": "/tmp/absl_testing", + "test_random_seed": 301, + "test_randomize_ordering_seed": "", + "xml_output_file": "", + "experimental_orbax_use_distributed_process_id": false, + "experimental_orbax_use_distributed_barrier": false, + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "workload": "fastmri", + "tuning_ruleset": "self", + "tuning_search_space": null, + "num_tuning_trials": 1, + "data_dir": "/data/fastmri", + "imagenet_v2_data_dir": null, + "librispeech_tokenizer_vocab_path": "", + "framework": "pytorch", + "torch_compile": true, + "experiment_dir": "/experiment_runs", + "experiment_name": "submissions_a100/ademamix_golden/study_1", + "save_checkpoints": false, + "save_intermediate_checkpoints": true, + "resume_last_run": null, + "append_timestamp": false, + "use_wandb": false, + "profile": false, + "max_global_steps": null, + "overwrite": true, + "hparam_start_index": null, + "hparam_end_index": null, + "rng_seed": -638637756, + "set_pytorch_max_split_size": false, + "pytorch_eval_num_workers": 0, + "capture_jax_trace": false, + "skip_evals": false, + "?": false, + "help": false, + "helpshort": false, + "helpfull": false, + "helpxml": false +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/trial_1/measurements.csv b/logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/trial_1/measurements.csv new file mode 100644 index 00000000..a5f43151 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/trial_1/measurements.csv @@ -0,0 +1,133 @@ +global_step,grad_norm,loss,train/ssim,train/loss,validation/ssim,validation/loss,validation/num_examples,test/ssim,test/loss,test/num_examples,score,total_duration,accumulated_submission_time,accumulated_eval_time,accumulated_logging_time,preemption_count +0,0.499999850988388,0.9429223537445068,,,,,,,,,,,,,, +1,,,0.2550915649959019,0.9745864868164062,0.2467691210058209,0.9774193278040588,3554.0,0.2693324979874249,0.976249432770176,3581.0,276.97007966041565,1026.6488287448883,276.97007966041565,749.0619850158691,0.0,0.0 +1,0.499999850988388,0.963639497756958,,,,,,,,,,,,,, +2,0.499999850988388,0.9868669509887696,,,,,,,,,,,,,, +3,0.499999850988388,1.014528751373291,,,,,,,,,,,,,, +4,0.499999850988388,0.9841161966323853,,,,,,,,,,,,,, +5,0.499999850988388,0.9266448616981506,,,,,,,,,,,,,, +6,0.499999850988388,0.913222074508667,,,,,,,,,,,,,, +7,0.499999850988388,1.0279455184936523,,,,,,,,,,,,,, +8,0.4999997913837433,1.0289382934570312,,,,,,,,,,,,,, +9,0.4999997913837433,0.9399406909942628,,,,,,,,,,,,,, +10,0.4999998211860657,0.875063419342041,,,,,,,,,,,,,, +11,0.4999998211860657,1.0047259330749512,,,,,,,,,,,,,, +12,0.4999997913837433,0.966424822807312,,,,,,,,,,,,,, +13,0.4999998807907104,0.9119424819946288,,,,,,,,,,,,,, +14,0.499999850988388,0.9280369877815248,,,,,,,,,,,,,, +15,0.499999850988388,0.910811185836792,,,,,,,,,,,,,, +16,0.4999998211860657,0.8996062874794006,,,,,,,,,,,,,, +17,0.499999850988388,0.8429145216941833,,,,,,,,,,,,,, +18,0.4999998807907104,0.8314876556396484,,,,,,,,,,,,,, +19,0.4999998807907104,0.8512351512908936,,,,,,,,,,,,,, +20,0.4999998807907104,0.7638193964958191,,,,,,,,,,,,,, +21,0.499999850988388,0.7625113725662231,,,,,,,,,,,,,, +22,0.4999998211860657,0.7678688168525696,,,,,,,,,,,,,, +23,0.4999998807907104,0.8291931748390198,,,,,,,,,,,,,, +24,0.499999850988388,0.7542852163314819,,,,,,,,,,,,,, +25,0.4999998211860657,0.777798056602478,,,,,,,,,,,,,, +26,0.4999998807907104,0.6588789224624634,,,,,,,,,,,,,, +27,0.4999999105930328,0.6972935199737549,,,,,,,,,,,,,, +28,0.4999999105930328,0.6838690042495728,,,,,,,,,,,,,, +29,0.499999850988388,0.6170892119407654,,,,,,,,,,,,,, +30,0.4999998807907104,0.585414707660675,,,,,,,,,,,,,, +31,0.4999998211860657,0.6080396175384521,,,,,,,,,,,,,, +32,0.499999850988388,0.5660892128944397,,,,,,,,,,,,,, +33,0.4999998211860657,0.5315609574317932,,,,,,,,,,,,,, +34,0.4999997913837433,0.5363882184028625,,,,,,,,,,,,,, +35,0.4999997615814209,0.5555329918861389,,,,,,,,,,,,,, +36,0.4999997317790985,0.5372941493988037,,,,,,,,,,,,,, +37,0.4999997317790985,0.5226091742515564,,,,,,,,,,,,,, +38,0.4999995827674866,0.5222477912902832,,,,,,,,,,,,,, +39,0.4999995827674866,0.5056538581848145,,,,,,,,,,,,,, +40,0.4999995231628418,0.4868140816688537,,,,,,,,,,,,,, +41,0.4999994337558746,0.4581905603408813,,,,,,,,,,,,,, +42,0.4999994337558746,0.4966820776462555,,,,,,,,,,,,,, +43,0.4999994039535522,0.4642952084541321,,,,,,,,,,,,,, +44,0.4999993741512298,0.4296906292438507,,,,,,,,,,,,,, +45,0.4999993443489074,0.4540170431137085,,,,,,,,,,,,,, +46,0.4999992847442627,0.4213637113571167,,,,,,,,,,,,,, +47,0.4999993443489074,0.4062718152999878,,,,,,,,,,,,,, +48,0.4999993443489074,0.3684569001197815,,,,,,,,,,,,,, +49,0.4999992847442627,0.3694145679473877,,,,,,,,,,,,,, +50,0.4999992847442627,0.3613159656524658,,,,,,,,,,,,,, +51,0.4999991953372955,0.3792106509208679,,,,,,,,,,,,,, +52,0.4999993145465851,0.4593162834644317,,,,,,,,,,,,,, +53,0.4999991953372955,0.4014071226119995,,,,,,,,,,,,,, +54,0.4999992847442627,0.3731710910797119,,,,,,,,,,,,,, +55,0.4999991357326507,0.3667851388454437,,,,,,,,,,,,,, +56,0.4999992251396179,0.4065838158130646,,,,,,,,,,,,,, +57,0.4999990165233612,0.3829172849655151,,,,,,,,,,,,,, +58,0.499999076128006,0.3716580271720886,,,,,,,,,,,,,, +59,0.4935530722141266,0.3010137677192688,,,,,,,,,,,,,, +60,0.4999989569187164,0.3832839727401733,,,,,,,,,,,,,, +61,0.4321922361850738,0.3691767752170563,,,,,,,,,,,,,, +62,0.3438910841941833,0.3615566194057464,,,,,,,,,,,,,, +63,0.3262908458709717,0.318778783082962,,,,,,,,,,,,,, +64,0.3027927577495575,0.3666422963142395,,,,,,,,,,,,,, +65,0.3887170255184173,0.4852113723754883,,,,,,,,,,,,,, +66,0.3767227232456207,0.4224021434783935,,,,,,,,,,,,,, +67,0.2699464857578277,0.3647130727767944,,,,,,,,,,,,,, +68,0.2301345616579055,0.2792384624481201,,,,,,,,,,,,,, +69,0.2257471680641174,0.3657369017601013,,,,,,,,,,,,,, +70,0.2214959412813186,0.3392394781112671,,,,,,,,,,,,,, +71,0.2082242667675018,0.363922655582428,,,,,,,,,,,,,, +72,0.1535502821207046,0.3572348356246948,,,,,,,,,,,,,, +73,0.3714425563812256,0.3967964351177215,,,,,,,,,,,,,, +74,0.1338373571634292,0.3268499970436096,,,,,,,,,,,,,, +75,0.4025216400623321,0.3745488822460174,,,,,,,,,,,,,, +76,0.1679157614707946,0.4113252162933349,,,,,,,,,,,,,, +77,0.3685870766639709,0.341490626335144,,,,,,,,,,,,,, +78,0.1800257563591003,0.406864583492279,,,,,,,,,,,,,, +79,0.2671121656894684,0.3959949910640716,,,,,,,,,,,,,, +80,0.1713919937610626,0.3329409956932068,,,,,,,,,,,,,, +81,0.1888335645198822,0.306530088186264,,,,,,,,,,,,,, +82,0.1475520133972168,0.3674258291721344,,,,,,,,,,,,,, +83,0.1607813686132431,0.3303079307079315,,,,,,,,,,,,,, +84,0.3284760415554046,0.396851897239685,,,,,,,,,,,,,, +85,0.2112756222486496,0.3251017928123474,,,,,,,,,,,,,, +86,0.1109224408864975,0.3151818811893463,,,,,,,,,,,,,, +87,0.2615417540073395,0.4413805603981018,,,,,,,,,,,,,, +88,0.1741317212581634,0.3504252433776855,,,,,,,,,,,,,, +89,0.153310090303421,0.3051583766937256,,,,,,,,,,,,,, +90,0.123948834836483,0.3468684554100036,,,,,,,,,,,,,, +91,0.167816087603569,0.3851467669010162,,,,,,,,,,,,,, +92,0.1961619108915329,0.3609141707420349,,,,,,,,,,,,,, +93,0.1937495470046997,0.4490894675254822,,,,,,,,,,,,,, +94,0.3534274697303772,0.3013468384742737,,,,,,,,,,,,,, +95,0.4999990463256836,0.4145954847335815,,,,,,,,,,,,,, +96,0.4110882580280304,0.3223589062690735,,,,,,,,,,,,,, +97,0.1635337769985199,0.2917133867740631,,,,,,,,,,,,,, +98,0.2026837915182113,0.3602484464645386,,,,,,,,,,,,,, +99,0.2178552597761154,0.3668801188468933,,,,,,,,,,,,,, +100,0.2202336192131042,0.2518127858638763,,,,,,,,,,,,,, +268,,,0.7188784054347447,0.2936295100620815,0.6924304865556415,0.3160394895056626,3554.0,0.7102057598916155,0.318160230971621,3581.0,387.70237612724304,1145.6982672214508,387.70237612724304,755.9916512966156,0.1044626235961914,0.0 +335,,,0.7157464708600726,0.2922060830252511,0.6924700546479319,0.3133579269177335,3554.0,0.709014031847773,0.3156670104588453,3581.0,499.1870410442352,1265.554081916809,499.1870410442352,763.003143787384,0.3435368537902832,0.0 +404,,,0.7273180144173759,0.2857339722769601,0.7019728540904615,0.3074569916819077,3554.0,0.7191851034539933,0.3096321487320231,3581.0,609.647013425827,1384.4995515346527,609.647013425827,770.3649923801422,0.3739564418792724,0.0 +474,,,0.7291603088378906,0.2834714310509817,0.7030225763444359,0.3052837348212401,3554.0,0.7202537726237433,0.3074292585324455,3581.0,719.0911104679108,1502.5156593322754,719.0911104679108,777.8156218528748,0.4036269187927246,0.0 +500,0.1116396933794021,0.3231690526008606,,,,,,,,,,,,,, +544,,,0.7331872667585101,0.2811053821018764,0.707059826955543,0.3029742909068127,3554.0,0.7242879220408056,0.3050175432185493,3581.0,828.81680727005,1620.7624669075012,828.81680727005,785.2173452377319,0.4332799911499023,0.0 +613,,,0.7336398533412388,0.2801985059465681,0.7073780890589828,0.301891457853387,3554.0,0.7245739231359956,0.3040621836602904,3581.0,938.8406875133514,1739.2497344017029,938.8406875133514,792.5633902549744,0.4639852046966553,0.0 +681,,,0.7347519738333566,0.2789818559374128,0.7084331007975169,0.3007541843257597,3554.0,0.7255433270865331,0.3028992943442823,3581.0,1048.9334847927094,1857.6184387207031,1048.9334847927094,799.6957902908325,0.5035965442657471,0.0 +747,,,0.7340025901794434,0.2780978339059012,0.7084184001521173,0.2996932991985263,3554.0,0.7255832104335381,0.3017607440964465,3581.0,1158.696293592453,1975.894044160843,1158.696293592453,807.0930454730988,0.5335135459899902,0.0 +820,,,0.737079279763358,0.2770411797932216,0.7108103462867543,0.2988100583189715,3554.0,0.7277975884005864,0.3009376472615889,3581.0,1269.4936056137085,2095.35284113884,1269.4936056137085,814.5684497356415,0.5636622905731201,0.0 +889,,,0.7376507350376674,0.2759171043123518,0.7114592355224747,0.2977601986757878,3554.0,0.728624434951829,0.2997621452631946,3581.0,1381.459115743637,2216.3427169322968,1381.459115743637,822.4694921970367,0.5941040515899658,0.0 +961,,,0.7378678321838379,0.2757465158190046,0.7114406192846089,0.2976615875707829,3554.0,0.7284563113044541,0.2996555851411966,3581.0,1490.9036588668823,2334.33344912529,1490.9036588668823,829.8707458972931,0.6444931030273438,0.0 +1000,0.1313855648040771,0.3523692786693573,,,,,,,,,,,,,, +1500,0.0762359276413917,0.2711002230644226,,,,,,,,,,,,,, +2000,0.0484939068555831,0.3641411662101745,,,,,,,,,,,,,, +2500,0.0825935453176498,0.3033905625343323,,,,,,,,,,,,,, +2927,,,0.7458196367536273,0.2680674280439104,0.7191054258845667,0.2905145665535312,3554.0,0.736318784797368,0.2920216398177883,3581.0,1599.721081495285,2452.672803878784,1599.721081495285,837.6486473083496,0.6740052700042725,0.0 +3000,0.0923818349838256,0.324034571647644,,,,,,,,,,,,,, +3500,0.17997507750988,0.2992158830165863,,,,,,,,,,,,,, +4000,0.2010032683610916,0.2885258197784424,,,,,,,,,,,,,, +4500,0.1886594146490097,0.2510321140289306,,,,,,,,,,,,,, +5000,0.0370094142854213,0.2419171780347824,,,,,,,,,,,,,, +5216,,,0.749497549874442,0.264472484588623,0.7226716375914463,0.2872891487804234,3554.0,0.7397906812866518,0.2887677382038013,3581.0,1708.4362881183624,2570.618153810501,1708.4362881183624,845.0584487915039,0.7028579711914062,0.0 +5500,0.0679423585534095,0.2743464708328247,,,,,,,,,,,,,, +6000,0.0931679233908653,0.375895619392395,,,,,,,,,,,,,, +6500,0.0987512618303299,0.2709999084472656,,,,,,,,,,,,,, +7000,0.0322979800403118,0.3085558116436004,,,,,,,,,,,,,, +7500,0.0426679141819477,0.2228668928146362,,,,,,,,,,,,,, +7507,,,0.751577513558524,0.2630536556243896,0.7241025462067389,0.2862729666671796,3554.0,0.7413039986430118,0.2875992243132156,3581.0,1817.1823732852936,2687.114144563675,1817.1823732852936,850.9800071716309,0.7312157154083252,0.0 +7507,,,,,,,,,,,1817.1823732852936,,,,,0.0 diff --git a/logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/trial_1/meta_data_0.json b/logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/trial_1/meta_data_0.json new file mode 100644 index 00000000..3edc1183 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/fastmri_pytorch/trial_1/meta_data_0.json @@ -0,0 +1,71 @@ +{ + "workload.eval_batch_size": 256, + "workload.eval_period_time_sec": 110, + "workload.max_allowed_runtime_sec": 2745, + "workload.num_channels": 32, + "workload.num_eval_train_examples": 3584, + "workload.num_pool_layers": 4, + "workload.num_test_examples": 3581, + "workload.num_train_examples": 34742, + "workload.num_validation_examples": 3554, + "workload.step_hint": 18094, + "workload.target_metric_name": "ssim", + "workload.test_target_value": 0.740633, + "workload.train_mean": 0.740633, + "workload.train_stddev": 0.740633, + "workload.use_layer_norm": false, + "workload.use_tanh": false, + "workload.validation_target_value": 0.723653, + "cpu.util.avg_percent_since_last": 4.3, + "cpu.freq.current": 2200.1959999999985, + "mem.total": 359053524992, + "mem.available": 349758210048, + "mem.used": 6139432960, + "mem.percent_used": 2.6, + "mem.read_bytes_since_boot": 18938061508096, + "mem.write_bytes_since_boot": 46846047744, + "net.bytes_sent_since_boot": 31308349, + "net.bytes_recv_since_boot": 31311442, + "gpu.count": 4, + "gpu.0.compute.util": 0.0, + "gpu.0.mem.util": 0.0288330078125, + "gpu.0.mem.total": 40960.0, + "gpu.0.mem.used": 1181.0, + "gpu.0.mem.free": 39146.0, + "gpu.0.temp.current": 34.0, + "gpu.1.compute.util": 0.0, + "gpu.1.mem.util": 0.0288330078125, + "gpu.1.mem.total": 40960.0, + "gpu.1.mem.used": 1181.0, + "gpu.1.mem.free": 39146.0, + "gpu.1.temp.current": 31.0, + "gpu.2.compute.util": 0.0, + "gpu.2.mem.util": 0.0288330078125, + "gpu.2.mem.total": 40960.0, + "gpu.2.mem.used": 1181.0, + "gpu.2.mem.free": 39146.0, + "gpu.2.temp.current": 31.0, + "gpu.3.compute.util": 0.0, + "gpu.3.mem.util": 0.0288330078125, + "gpu.3.mem.total": 40960.0, + "gpu.3.mem.used": 1181.0, + "gpu.3.mem.free": 39146.0, + "gpu.3.temp.current": 33.0, + "gpu.avg.compute.util": 0.0, + "gpu.avg.mem.util": 0.0288330078125, + "gpu.avg.mem.total": 40960.0, + "gpu.avg.mem.used": 1181.0, + "gpu.avg.mem.free": 39146.0, + "gpu.avg.temp.current": 32.25, + "os_platform": "Linux-6.1.0-44-cloud-amd64-x86_64-with-glibc2.31", + "python_version": "3.11.10", + "python_compiler": "GCC 9.4.0", + "git_branch": "main", + "git_commit_hash": "b21be29be0a1573fb4f78f849aea019cdb520862", + "cpu_model_name": "Intel(R) Xeon(R) CPU @ 2.20GHz", + "cpu_count": 24, + "gpu_model_name": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_driver": "550.90.12", + "rng_seed": -638637756 +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_1/finewebedu_lm_pytorch/finewebedu_lm_pytorch_09-11-2026-14-45-42.log b/logs/self_tuning/ademamix_golden/study_1/finewebedu_lm_pytorch/finewebedu_lm_pytorch_09-11-2026-14-45-42.log new file mode 100644 index 00000000..08a87a68 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/finewebedu_lm_pytorch/finewebedu_lm_pytorch_09-11-2026-14-45-42.log @@ -0,0 +1,470 @@ +torchrun --redirects 1:0,2:0,3:0 --standalone --nnodes=1 --nproc_per_node=4 submission_runner.py --framework=pytorch --workload=finewebedu_lm --submission_path=submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py --data_dir=/data/fineweb_edu_10B --experiment_dir=/experiment_runs --experiment_name=submissions_a100/ademamix_golden/study_1 --overwrite=True --save_checkpoints=False --rng_seed=140606013 --torch_compile=true --tuning_ruleset=self 2>&1 | tee -a /logs/finewebedu_lm_pytorch_09-11-2026-14-45-42.log +W0911 14:45:44.552000 9 site-packages/torch/distributed/run.py:803] +W0911 14:45:44.552000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0911 14:45:44.552000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0911 14:45:44.552000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-11 14:45:46.177368: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-11 14:45:46.177367: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-11 14:45:46.177368: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-11 14:45:46.177387: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789137946.198000 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789137946.197996 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789137946.198000 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789137946.197996 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789137946.204771 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789137946.204774 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789137946.204778 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789137946.204783 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789137946.221669 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137946.221669 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137946.221665 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137946.221689 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137946.221689 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137946.221672 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137946.221692 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137946.221692 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137946.221694 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137946.221694 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137946.221694 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137946.221694 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137946.221696 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137946.221696 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137946.221698 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137946.221699 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137951.778217 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789137951.838021 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789137951.933374 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789137952.025436 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W911 14:45:53.209046672 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0911 14:45:54.824723 140605188629696 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/finewebedu_lm_pytorch. +I0911 14:45:54.824722 140560192804032 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/finewebedu_lm_pytorch. +I0911 14:45:54.824722 139757995185344 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/finewebedu_lm_pytorch. +I0911 14:45:54.824744 140604651087040 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/finewebedu_lm_pytorch. +I0911 14:45:54.857185 140605188629696 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/finewebedu_lm_pytorch/trial_1. +I0911 14:45:55.107672 140605188629696 submission_runner.py:242] Initializing dataset. +I0911 14:45:55.107837 140605188629696 submission_runner.py:251] Initializing model. +I0911 14:46:01.531689 140605188629696 submission_runner.py:290] Performing `torch.compile`. +I0911 14:46:02.908406 140560192804032 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0911 14:46:02.908570 140560192804032 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0911 14:46:02.912361 140605188629696 submission_runner.py:294] Initializing optimizer. +I0911 14:46:02.913087 140605188629696 submission_runner.py:299] Initializing metrics bundle. +I0911 14:46:02.913080 140604651087040 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0911 14:46:02.913122 139757995185344 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0911 14:46:02.913231 140604651087040 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0911 14:46:02.913233 140605188629696 submission_runner.py:321] Initializing checkpoint and logger. +I0911 14:46:02.913269 139757995185344 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0911 14:46:02.913636 140605188629696 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_1/finewebedu_lm_pytorch/trial_1/meta_data_0.json. +I0911 14:46:02.913811 140605188629696 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0911 14:46:02.913866 140605188629696 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0911 14:46:03.175228 140605188629696 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_1/finewebedu_lm_pytorch/trial_1/flags_0.json. +I0911 14:46:03.189666 140605188629696 submission_runner.py:359] Starting training loop. +[rank0]:W0911 14:46:05.238000 38 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank1]:W0911 14:46:05.274000 39 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank2]:W0911 14:46:05.307000 40 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank3]:W0911 14:46:05.355000 41 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +I0911 14:46:47.795981 140578960201472 logging_writer.py:48] [0] global_step=0, grad_norm=0.5, loss=11.0153 +I0911 14:46:47.824700 140605188629696 submission.py:307] 0) loss = 11.015, grad_norm = 0.500 +I0911 14:46:48.405434 140605188629696 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +I0911 14:47:44.491569 140605188629696 spec.py:346] Evaluating on the validation split. +I0911 14:50:35.075180 140605188629696 spec.py:363] Evaluating on the test split. +I0911 14:50:35.075823 140605188629696 submission_runner.py:516] Time since start: 271.89s, Step: 1, {'train/loss': 11.020582580566407, 'train/ppl': 61119.27614337982, 'validation/loss': 11.021556805466751, 'validation/ppl': 61178.84907809615, 'validation/num_examples': 100000, 'test/loss': 0.0, 'test/ppl': 1.0, 'test/num_examples': 0, 'score': 44.63611102104187, 'total_duration': 271.8861677646637, 'accumulated_submission_time': 44.63611102104187, 'accumulated_eval_time': 226.6703073978424, 'accumulated_logging_time': 0} +I0911 14:50:35.093395 140569892026112 logging_writer.py:48] [1] accumulated_eval_time=226.67, accumulated_logging_time=0, accumulated_submission_time=44.6361, global_step=1, preemption_count=0, score=44.6361, test/loss=0, test/num_examples=0, test/ppl=1, total_duration=271.886, train/loss=11.0206, train/ppl=61119.3, validation/loss=11.0216, validation/num_examples=100000, validation/ppl=61178.8 +I0911 14:50:36.756534 140569883633408 logging_writer.py:48] [1] global_step=1, grad_norm=0.5, loss=11.0151 +I0911 14:50:36.759411 140605188629696 submission.py:307] 1) loss = 11.015, grad_norm = 0.500 +I0911 14:50:37.200394 140569892026112 logging_writer.py:48] [2] global_step=2, grad_norm=0.5, loss=11.0068 +I0911 14:50:37.203278 140605188629696 submission.py:307] 2) loss = 11.007, grad_norm = 0.500 +I0911 14:50:37.634511 140569883633408 logging_writer.py:48] [3] global_step=3, grad_norm=0.5, loss=10.9747 +I0911 14:50:37.637282 140605188629696 submission.py:307] 3) loss = 10.975, grad_norm = 0.500 +I0911 14:50:38.067003 140569892026112 logging_writer.py:48] [4] global_step=4, grad_norm=0.5, loss=10.9318 +I0911 14:50:38.069793 140605188629696 submission.py:307] 4) loss = 10.932, grad_norm = 0.500 +I0911 14:50:38.499360 140569883633408 logging_writer.py:48] [5] global_step=5, grad_norm=0.5, loss=10.8632 +I0911 14:50:38.502210 140605188629696 submission.py:307] 5) loss = 10.863, grad_norm = 0.500 +I0911 14:50:38.930762 140569892026112 logging_writer.py:48] [6] global_step=6, grad_norm=0.5, loss=10.7872 +I0911 14:50:38.933593 140605188629696 submission.py:307] 6) loss = 10.787, grad_norm = 0.500 +I0911 14:50:39.363937 140569883633408 logging_writer.py:48] [7] global_step=7, grad_norm=0.5, loss=10.7019 +I0911 14:50:39.366706 140605188629696 submission.py:307] 7) loss = 10.702, grad_norm = 0.500 +I0911 14:50:39.795747 140569892026112 logging_writer.py:48] [8] global_step=8, grad_norm=0.5, loss=10.6166 +I0911 14:50:39.798572 140605188629696 submission.py:307] 8) loss = 10.617, grad_norm = 0.500 +I0911 14:50:40.226795 140569883633408 logging_writer.py:48] [9] global_step=9, grad_norm=0.5, loss=10.4723 +I0911 14:50:40.229574 140605188629696 submission.py:307] 9) loss = 10.472, grad_norm = 0.500 +I0911 14:50:40.659706 140569892026112 logging_writer.py:48] [10] global_step=10, grad_norm=0.5, loss=10.3515 +I0911 14:50:40.662519 140605188629696 submission.py:307] 10) loss = 10.352, grad_norm = 0.500 +I0911 14:50:41.091380 140569883633408 logging_writer.py:48] [11] global_step=11, grad_norm=0.5, loss=10.2401 +I0911 14:50:41.094169 140605188629696 submission.py:307] 11) loss = 10.240, grad_norm = 0.500 +I0911 14:50:41.523325 140569892026112 logging_writer.py:48] [12] global_step=12, grad_norm=0.5, loss=10.1202 +I0911 14:50:41.526176 140605188629696 submission.py:307] 12) loss = 10.120, grad_norm = 0.500 +I0911 14:50:41.955276 140569883633408 logging_writer.py:48] [13] global_step=13, grad_norm=0.5, loss=9.99427 +I0911 14:50:41.958129 140605188629696 submission.py:307] 13) loss = 9.994, grad_norm = 0.500 +I0911 14:50:42.384059 140569892026112 logging_writer.py:48] [14] global_step=14, grad_norm=0.5, loss=9.92115 +I0911 14:50:42.386841 140605188629696 submission.py:307] 14) loss = 9.921, grad_norm = 0.500 +I0911 14:50:42.814636 140569883633408 logging_writer.py:48] [15] global_step=15, grad_norm=0.5, loss=9.7727 +I0911 14:50:42.817377 140605188629696 submission.py:307] 15) loss = 9.773, grad_norm = 0.500 +I0911 14:50:43.246764 140569892026112 logging_writer.py:48] [16] global_step=16, grad_norm=0.5, loss=9.74113 +I0911 14:50:43.249567 140605188629696 submission.py:307] 16) loss = 9.741, grad_norm = 0.500 +I0911 14:50:43.679161 140569883633408 logging_writer.py:48] [17] global_step=17, grad_norm=0.5, loss=9.62004 +I0911 14:50:43.682042 140605188629696 submission.py:307] 17) loss = 9.620, grad_norm = 0.500 +I0911 14:50:44.109508 140569892026112 logging_writer.py:48] [18] global_step=18, grad_norm=0.5, loss=9.55253 +I0911 14:50:44.112330 140605188629696 submission.py:307] 18) loss = 9.553, grad_norm = 0.500 +I0911 14:50:44.541690 140569883633408 logging_writer.py:48] [19] global_step=19, grad_norm=0.5, loss=9.44586 +I0911 14:50:44.544587 140605188629696 submission.py:307] 19) loss = 9.446, grad_norm = 0.500 +I0911 14:50:44.973516 140569892026112 logging_writer.py:48] [20] global_step=20, grad_norm=0.5, loss=9.42764 +I0911 14:50:44.976255 140605188629696 submission.py:307] 20) loss = 9.428, grad_norm = 0.500 +I0911 14:50:45.401927 140569883633408 logging_writer.py:48] [21] global_step=21, grad_norm=0.5, loss=9.4021 +I0911 14:50:45.404754 140605188629696 submission.py:307] 21) loss = 9.402, grad_norm = 0.500 +I0911 14:50:45.831952 140569892026112 logging_writer.py:48] [22] global_step=22, grad_norm=0.5, loss=9.37511 +I0911 14:50:45.834787 140605188629696 submission.py:307] 22) loss = 9.375, grad_norm = 0.500 +I0911 14:50:46.263898 140569883633408 logging_writer.py:48] [23] global_step=23, grad_norm=0.5, loss=9.33595 +I0911 14:50:46.266784 140605188629696 submission.py:307] 23) loss = 9.336, grad_norm = 0.500 +I0911 14:50:46.695830 140569892026112 logging_writer.py:48] [24] global_step=24, grad_norm=0.5, loss=9.31078 +I0911 14:50:46.698683 140605188629696 submission.py:307] 24) loss = 9.311, grad_norm = 0.500 +I0911 14:50:47.126183 140569883633408 logging_writer.py:48] [25] global_step=25, grad_norm=0.5, loss=9.26592 +I0911 14:50:47.129104 140605188629696 submission.py:307] 25) loss = 9.266, grad_norm = 0.500 +I0911 14:50:47.558939 140569892026112 logging_writer.py:48] [26] global_step=26, grad_norm=0.5, loss=9.24418 +I0911 14:50:47.561822 140605188629696 submission.py:307] 26) loss = 9.244, grad_norm = 0.500 +I0911 14:50:47.990486 140569883633408 logging_writer.py:48] [27] global_step=27, grad_norm=0.5, loss=9.22173 +I0911 14:50:47.993245 140605188629696 submission.py:307] 27) loss = 9.222, grad_norm = 0.500 +I0911 14:50:48.420762 140569892026112 logging_writer.py:48] [28] global_step=28, grad_norm=0.5, loss=9.21657 +I0911 14:50:48.423610 140605188629696 submission.py:307] 28) loss = 9.217, grad_norm = 0.500 +I0911 14:50:48.850886 140569883633408 logging_writer.py:48] [29] global_step=29, grad_norm=0.5, loss=9.2283 +I0911 14:50:48.853735 140605188629696 submission.py:307] 29) loss = 9.228, grad_norm = 0.500 +I0911 14:50:49.283062 140569892026112 logging_writer.py:48] [30] global_step=30, grad_norm=0.5, loss=9.17224 +I0911 14:50:49.285934 140605188629696 submission.py:307] 30) loss = 9.172, grad_norm = 0.500 +I0911 14:50:49.714000 140569883633408 logging_writer.py:48] [31] global_step=31, grad_norm=0.5, loss=9.17262 +I0911 14:50:49.716825 140605188629696 submission.py:307] 31) loss = 9.173, grad_norm = 0.500 +I0911 14:50:50.144859 140569892026112 logging_writer.py:48] [32] global_step=32, grad_norm=0.5, loss=9.13588 +I0911 14:50:50.147669 140605188629696 submission.py:307] 32) loss = 9.136, grad_norm = 0.500 +I0911 14:50:50.577360 140569883633408 logging_writer.py:48] [33] global_step=33, grad_norm=0.5, loss=9.11179 +I0911 14:50:50.580165 140605188629696 submission.py:307] 33) loss = 9.112, grad_norm = 0.500 +I0911 14:50:51.008075 140569892026112 logging_writer.py:48] [34] global_step=34, grad_norm=0.5, loss=9.04943 +I0911 14:50:51.010883 140605188629696 submission.py:307] 34) loss = 9.049, grad_norm = 0.500 +I0911 14:50:51.439196 140569883633408 logging_writer.py:48] [35] global_step=35, grad_norm=0.5, loss=9.0421 +I0911 14:50:51.442065 140605188629696 submission.py:307] 35) loss = 9.042, grad_norm = 0.500 +I0911 14:50:51.871836 140569892026112 logging_writer.py:48] [36] global_step=36, grad_norm=0.5, loss=9.06561 +I0911 14:50:51.874871 140605188629696 submission.py:307] 36) loss = 9.066, grad_norm = 0.500 +I0911 14:50:52.301687 140569883633408 logging_writer.py:48] [37] global_step=37, grad_norm=0.5, loss=9.04134 +I0911 14:50:52.304533 140605188629696 submission.py:307] 37) loss = 9.041, grad_norm = 0.500 +I0911 14:50:52.731751 140569892026112 logging_writer.py:48] [38] global_step=38, grad_norm=0.5, loss=8.93473 +I0911 14:50:52.734545 140605188629696 submission.py:307] 38) loss = 8.935, grad_norm = 0.500 +I0911 14:50:53.161704 140569883633408 logging_writer.py:48] [39] global_step=39, grad_norm=0.5, loss=8.93584 +I0911 14:50:53.164508 140605188629696 submission.py:307] 39) loss = 8.936, grad_norm = 0.500 +I0911 14:50:53.593185 140569892026112 logging_writer.py:48] [40] global_step=40, grad_norm=0.5, loss=8.92206 +I0911 14:50:53.596005 140605188629696 submission.py:307] 40) loss = 8.922, grad_norm = 0.500 +I0911 14:50:54.024350 140569883633408 logging_writer.py:48] [41] global_step=41, grad_norm=0.5, loss=8.90531 +I0911 14:50:54.027151 140605188629696 submission.py:307] 41) loss = 8.905, grad_norm = 0.500 +I0911 14:50:54.454741 140569892026112 logging_writer.py:48] [42] global_step=42, grad_norm=0.5, loss=8.87587 +I0911 14:50:54.457568 140605188629696 submission.py:307] 42) loss = 8.876, grad_norm = 0.500 +I0911 14:50:54.887065 140569883633408 logging_writer.py:48] [43] global_step=43, grad_norm=0.5, loss=8.83646 +I0911 14:50:54.889881 140605188629696 submission.py:307] 43) loss = 8.836, grad_norm = 0.500 +I0911 14:50:55.318535 140569892026112 logging_writer.py:48] [44] global_step=44, grad_norm=0.5, loss=8.84478 +I0911 14:50:55.321339 140605188629696 submission.py:307] 44) loss = 8.845, grad_norm = 0.500 +I0911 14:50:55.747643 140569883633408 logging_writer.py:48] [45] global_step=45, grad_norm=0.5, loss=8.77143 +I0911 14:50:55.750462 140605188629696 submission.py:307] 45) loss = 8.771, grad_norm = 0.500 +I0911 14:50:56.177850 140569892026112 logging_writer.py:48] [46] global_step=46, grad_norm=0.5, loss=8.77482 +I0911 14:50:56.180664 140605188629696 submission.py:307] 46) loss = 8.775, grad_norm = 0.500 +I0911 14:50:56.609332 140569883633408 logging_writer.py:48] [47] global_step=47, grad_norm=0.5, loss=8.78464 +I0911 14:50:56.612028 140605188629696 submission.py:307] 47) loss = 8.785, grad_norm = 0.500 +I0911 14:50:57.040713 140569892026112 logging_writer.py:48] [48] global_step=48, grad_norm=0.5, loss=8.67914 +I0911 14:50:57.043490 140605188629696 submission.py:307] 48) loss = 8.679, grad_norm = 0.500 +I0911 14:50:57.471522 140569883633408 logging_writer.py:48] [49] global_step=49, grad_norm=0.5, loss=8.68061 +I0911 14:50:57.474290 140605188629696 submission.py:307] 49) loss = 8.681, grad_norm = 0.500 +I0911 14:50:57.902625 140569892026112 logging_writer.py:48] [50] global_step=50, grad_norm=0.5, loss=8.69974 +I0911 14:50:57.905479 140605188629696 submission.py:307] 50) loss = 8.700, grad_norm = 0.500 +I0911 14:50:58.334859 140569883633408 logging_writer.py:48] [51] global_step=51, grad_norm=0.5, loss=8.66935 +I0911 14:50:58.337692 140605188629696 submission.py:307] 51) loss = 8.669, grad_norm = 0.500 +I0911 14:50:58.765179 140569892026112 logging_writer.py:48] [52] global_step=52, grad_norm=0.5, loss=8.63885 +I0911 14:50:58.767954 140605188629696 submission.py:307] 52) loss = 8.639, grad_norm = 0.500 +I0911 14:50:59.194046 140569883633408 logging_writer.py:48] [53] global_step=53, grad_norm=0.5, loss=8.62476 +I0911 14:50:59.196886 140605188629696 submission.py:307] 53) loss = 8.625, grad_norm = 0.500 +I0911 14:50:59.626007 140569892026112 logging_writer.py:48] [54] global_step=54, grad_norm=0.5, loss=8.57328 +I0911 14:50:59.628781 140605188629696 submission.py:307] 54) loss = 8.573, grad_norm = 0.500 +I0911 14:51:00.058604 140569883633408 logging_writer.py:48] [55] global_step=55, grad_norm=0.5, loss=8.5828 +I0911 14:51:00.061356 140605188629696 submission.py:307] 55) loss = 8.583, grad_norm = 0.500 +I0911 14:51:00.489238 140569892026112 logging_writer.py:48] [56] global_step=56, grad_norm=0.5, loss=8.61161 +I0911 14:51:00.492043 140605188629696 submission.py:307] 56) loss = 8.612, grad_norm = 0.500 +I0911 14:51:00.921467 140569883633408 logging_writer.py:48] [57] global_step=57, grad_norm=0.5, loss=8.52864 +I0911 14:51:00.924199 140605188629696 submission.py:307] 57) loss = 8.529, grad_norm = 0.500 +I0911 14:51:01.353186 140569892026112 logging_writer.py:48] [58] global_step=58, grad_norm=0.5, loss=8.5014 +I0911 14:51:01.355965 140605188629696 submission.py:307] 58) loss = 8.501, grad_norm = 0.500 +I0911 14:51:01.783003 140569883633408 logging_writer.py:48] [59] global_step=59, grad_norm=0.5, loss=8.51353 +I0911 14:51:01.785826 140605188629696 submission.py:307] 59) loss = 8.514, grad_norm = 0.500 +I0911 14:51:02.211966 140569892026112 logging_writer.py:48] [60] global_step=60, grad_norm=0.5, loss=8.52962 +I0911 14:51:02.214743 140605188629696 submission.py:307] 60) loss = 8.530, grad_norm = 0.500 +I0911 14:51:02.643243 140569883633408 logging_writer.py:48] [61] global_step=61, grad_norm=0.5, loss=8.45072 +I0911 14:51:02.646084 140605188629696 submission.py:307] 61) loss = 8.451, grad_norm = 0.500 +I0911 14:51:03.074937 140569892026112 logging_writer.py:48] [62] global_step=62, grad_norm=0.5, loss=8.44301 +I0911 14:51:03.077818 140605188629696 submission.py:307] 62) loss = 8.443, grad_norm = 0.500 +I0911 14:51:03.505626 140569883633408 logging_writer.py:48] [63] global_step=63, grad_norm=0.5, loss=8.39296 +I0911 14:51:03.508368 140605188629696 submission.py:307] 63) loss = 8.393, grad_norm = 0.500 +I0911 14:51:03.937076 140569892026112 logging_writer.py:48] [64] global_step=64, grad_norm=0.5, loss=8.41195 +I0911 14:51:03.939859 140605188629696 submission.py:307] 64) loss = 8.412, grad_norm = 0.500 +I0911 14:51:04.368379 140569883633408 logging_writer.py:48] [65] global_step=65, grad_norm=0.5, loss=8.40928 +I0911 14:51:04.371533 140605188629696 submission.py:307] 65) loss = 8.409, grad_norm = 0.500 +I0911 14:51:04.799738 140569892026112 logging_writer.py:48] [66] global_step=66, grad_norm=0.5, loss=8.30307 +I0911 14:51:04.802520 140605188629696 submission.py:307] 66) loss = 8.303, grad_norm = 0.500 +I0911 14:51:05.229948 140569883633408 logging_writer.py:48] [67] global_step=67, grad_norm=0.5, loss=8.28684 +I0911 14:51:05.232806 140605188629696 submission.py:307] 67) loss = 8.287, grad_norm = 0.500 +I0911 14:51:05.660399 140569892026112 logging_writer.py:48] [68] global_step=68, grad_norm=0.5, loss=8.30109 +I0911 14:51:05.663237 140605188629696 submission.py:307] 68) loss = 8.301, grad_norm = 0.500 +I0911 14:51:06.092078 140569883633408 logging_writer.py:48] [69] global_step=69, grad_norm=0.5, loss=8.35445 +I0911 14:51:06.095016 140605188629696 submission.py:307] 69) loss = 8.354, grad_norm = 0.500 +I0911 14:51:06.523581 140569892026112 logging_writer.py:48] [70] global_step=70, grad_norm=0.5, loss=8.27059 +I0911 14:51:06.526403 140605188629696 submission.py:307] 70) loss = 8.271, grad_norm = 0.500 +I0911 14:51:06.953949 140569883633408 logging_writer.py:48] [71] global_step=71, grad_norm=0.5, loss=8.2759 +I0911 14:51:06.956918 140605188629696 submission.py:307] 71) loss = 8.276, grad_norm = 0.500 +I0911 14:51:07.385637 140569892026112 logging_writer.py:48] [72] global_step=72, grad_norm=0.5, loss=8.25628 +I0911 14:51:07.388430 140605188629696 submission.py:307] 72) loss = 8.256, grad_norm = 0.500 +I0911 14:51:07.817222 140569883633408 logging_writer.py:48] [73] global_step=73, grad_norm=0.5, loss=8.25063 +I0911 14:51:07.820018 140605188629696 submission.py:307] 73) loss = 8.251, grad_norm = 0.500 +I0911 14:51:08.247702 140569892026112 logging_writer.py:48] [74] global_step=74, grad_norm=0.5, loss=8.25846 +I0911 14:51:08.250519 140605188629696 submission.py:307] 74) loss = 8.258, grad_norm = 0.500 +I0911 14:51:08.679686 140569883633408 logging_writer.py:48] [75] global_step=75, grad_norm=0.5, loss=8.23592 +I0911 14:51:08.682894 140605188629696 submission.py:307] 75) loss = 8.236, grad_norm = 0.500 +I0911 14:51:09.108979 140569892026112 logging_writer.py:48] [76] global_step=76, grad_norm=0.5, loss=8.12345 +I0911 14:51:09.111845 140605188629696 submission.py:307] 76) loss = 8.123, grad_norm = 0.500 +I0911 14:51:09.539547 140569883633408 logging_writer.py:48] [77] global_step=77, grad_norm=0.5, loss=8.13457 +I0911 14:51:09.542470 140605188629696 submission.py:307] 77) loss = 8.135, grad_norm = 0.500 +I0911 14:51:09.971017 140569892026112 logging_writer.py:48] [78] global_step=78, grad_norm=0.5, loss=8.10833 +I0911 14:51:09.973851 140605188629696 submission.py:307] 78) loss = 8.108, grad_norm = 0.500 +I0911 14:51:10.403123 140569883633408 logging_writer.py:48] [79] global_step=79, grad_norm=0.5, loss=8.12142 +I0911 14:51:10.405914 140605188629696 submission.py:307] 79) loss = 8.121, grad_norm = 0.500 +I0911 14:51:10.833866 140569892026112 logging_writer.py:48] [80] global_step=80, grad_norm=0.5, loss=8.01304 +I0911 14:51:10.836631 140605188629696 submission.py:307] 80) loss = 8.013, grad_norm = 0.500 +I0911 14:51:11.263652 140569883633408 logging_writer.py:48] [81] global_step=81, grad_norm=0.5, loss=8.01638 +I0911 14:51:11.266480 140605188629696 submission.py:307] 81) loss = 8.016, grad_norm = 0.500 +I0911 14:51:11.695984 140569892026112 logging_writer.py:48] [82] global_step=82, grad_norm=0.5, loss=8.00387 +I0911 14:51:11.698790 140605188629696 submission.py:307] 82) loss = 8.004, grad_norm = 0.500 +I0911 14:51:12.126348 140569883633408 logging_writer.py:48] [83] global_step=83, grad_norm=0.5, loss=8.02026 +I0911 14:51:12.129182 140605188629696 submission.py:307] 83) loss = 8.020, grad_norm = 0.500 +I0911 14:51:12.556144 140569892026112 logging_writer.py:48] [84] global_step=84, grad_norm=0.5, loss=8.00921 +I0911 14:51:12.558917 140605188629696 submission.py:307] 84) loss = 8.009, grad_norm = 0.500 +I0911 14:51:12.986047 140569883633408 logging_writer.py:48] [85] global_step=85, grad_norm=0.5, loss=7.9704 +I0911 14:51:12.988853 140605188629696 submission.py:307] 85) loss = 7.970, grad_norm = 0.500 +I0911 14:51:13.417350 140569892026112 logging_writer.py:48] [86] global_step=86, grad_norm=0.5, loss=7.96443 +I0911 14:51:13.420131 140605188629696 submission.py:307] 86) loss = 7.964, grad_norm = 0.500 +I0911 14:51:13.847909 140569883633408 logging_writer.py:48] [87] global_step=87, grad_norm=0.5, loss=7.97614 +I0911 14:51:13.850752 140605188629696 submission.py:307] 87) loss = 7.976, grad_norm = 0.500 +I0911 14:51:14.278088 140569892026112 logging_writer.py:48] [88] global_step=88, grad_norm=0.5, loss=7.94269 +I0911 14:51:14.280890 140605188629696 submission.py:307] 88) loss = 7.943, grad_norm = 0.500 +I0911 14:51:14.709327 140569883633408 logging_writer.py:48] [89] global_step=89, grad_norm=0.5, loss=7.86316 +I0911 14:51:14.712117 140605188629696 submission.py:307] 89) loss = 7.863, grad_norm = 0.500 +I0911 14:51:15.141228 140569892026112 logging_writer.py:48] [90] global_step=90, grad_norm=0.5, loss=7.86102 +I0911 14:51:15.144022 140605188629696 submission.py:307] 90) loss = 7.861, grad_norm = 0.500 +I0911 14:51:15.571384 140569883633408 logging_writer.py:48] [91] global_step=91, grad_norm=0.5, loss=7.84709 +I0911 14:51:15.574166 140605188629696 submission.py:307] 91) loss = 7.847, grad_norm = 0.500 +I0911 14:51:16.000301 140569892026112 logging_writer.py:48] [92] global_step=92, grad_norm=0.5, loss=7.82293 +I0911 14:51:16.003464 140605188629696 submission.py:307] 92) loss = 7.823, grad_norm = 0.500 +I0911 14:51:16.432096 140569883633408 logging_writer.py:48] [93] global_step=93, grad_norm=0.5, loss=7.78815 +I0911 14:51:16.434893 140605188629696 submission.py:307] 93) loss = 7.788, grad_norm = 0.500 +I0911 14:51:16.865039 140569892026112 logging_writer.py:48] [94] global_step=94, grad_norm=0.5, loss=7.80946 +I0911 14:51:16.867861 140605188629696 submission.py:307] 94) loss = 7.809, grad_norm = 0.500 +I0911 14:51:17.296267 140569883633408 logging_writer.py:48] [95] global_step=95, grad_norm=0.5, loss=7.76081 +I0911 14:51:17.299304 140605188629696 submission.py:307] 95) loss = 7.761, grad_norm = 0.500 +I0911 14:51:17.727354 140569892026112 logging_writer.py:48] [96] global_step=96, grad_norm=0.5, loss=7.74224 +I0911 14:51:17.730212 140605188629696 submission.py:307] 96) loss = 7.742, grad_norm = 0.500 +I0911 14:51:18.158979 140569883633408 logging_writer.py:48] [97] global_step=97, grad_norm=0.5, loss=7.71643 +I0911 14:51:18.161793 140605188629696 submission.py:307] 97) loss = 7.716, grad_norm = 0.500 +I0911 14:51:18.590122 140569892026112 logging_writer.py:48] [98] global_step=98, grad_norm=0.5, loss=7.69905 +I0911 14:51:18.593001 140605188629696 submission.py:307] 98) loss = 7.699, grad_norm = 0.500 +I0911 14:51:19.019277 140569883633408 logging_writer.py:48] [99] global_step=99, grad_norm=0.5, loss=7.67196 +I0911 14:51:19.022378 140605188629696 submission.py:307] 99) loss = 7.672, grad_norm = 0.500 +I0911 14:51:19.449582 140569892026112 logging_writer.py:48] [100] global_step=100, grad_norm=0.5, loss=7.77724 +I0911 14:51:19.452333 140605188629696 submission.py:307] 100) loss = 7.777, grad_norm = 0.500 +I0911 14:54:06.846545 140569883633408 logging_writer.py:48] [500] global_step=500, grad_norm=0.499999, loss=5.57861 +I0911 14:54:06.849542 140605188629696 submission.py:307] 500) loss = 5.579, grad_norm = 0.500 +I0911 14:57:36.536587 140569892026112 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.499999, loss=4.59631 +I0911 14:57:36.539687 140605188629696 submission.py:307] 1000) loss = 4.596, grad_norm = 0.500 +I0911 15:01:06.305680 140569883633408 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.390261, loss=4.12502 +I0911 15:01:06.308867 140605188629696 submission.py:307] 1500) loss = 4.125, grad_norm = 0.390 +I0911 15:04:36.127688 140569892026112 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.317957, loss=3.93876 +I0911 15:04:36.130880 140605188629696 submission.py:307] 2000) loss = 3.939, grad_norm = 0.318 +I0911 15:08:05.995120 140569883633408 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.388366, loss=3.77712 +I0911 15:08:05.998355 140605188629696 submission.py:307] 2500) loss = 3.777, grad_norm = 0.388 +I0911 15:11:35.904386 140569892026112 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.284542, loss=3.68835 +I0911 15:11:35.907642 140605188629696 submission.py:307] 3000) loss = 3.688, grad_norm = 0.285 +I0911 15:15:05.937580 140569883633408 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.263329, loss=3.68256 +I0911 15:15:05.941221 140605188629696 submission.py:307] 3500) loss = 3.683, grad_norm = 0.263 +I0911 15:18:35.800714 140569892026112 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.241085, loss=3.58217 +I0911 15:18:35.803796 140605188629696 submission.py:307] 4000) loss = 3.582, grad_norm = 0.241 +I0911 15:22:05.698681 140569883633408 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.217426, loss=3.57443 +I0911 15:22:05.701941 140605188629696 submission.py:307] 4500) loss = 3.574, grad_norm = 0.217 +I0911 15:25:35.659200 140569892026112 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.208807, loss=3.55145 +I0911 15:25:35.662385 140605188629696 submission.py:307] 5000) loss = 3.551, grad_norm = 0.209 +I0911 15:29:05.597716 140569883633408 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.194293, loss=3.53429 +I0911 15:29:05.600893 140605188629696 submission.py:307] 5500) loss = 3.534, grad_norm = 0.194 +I0911 15:32:35.449307 140569892026112 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.189947, loss=3.4752 +I0911 15:32:35.452645 140605188629696 submission.py:307] 6000) loss = 3.475, grad_norm = 0.190 +I0911 15:33:26.802569 140605188629696 spec.py:333] Evaluating on the training split. +I0911 15:33:44.555924 140605188629696 spec.py:346] Evaluating on the validation split. +I0911 15:36:35.676573 140605188629696 spec.py:363] Evaluating on the test split. +I0911 15:36:35.677244 140605188629696 submission_runner.py:516] Time since start: 3032.49s, Step: 6122, {'train/loss': 3.4746829986572267, 'train/ppl': 32.28759170422644, 'validation/loss': 3.4671412319173593, 'validation/ppl': 32.04500214577074, 'validation/num_examples': 100000, 'test/loss': 0.0, 'test/ppl': 1.0, 'test/num_examples': 0, 'score': 2612.907131433487, 'total_duration': 3032.4875638484955, 'accumulated_submission_time': 2612.907131433487, 'accumulated_eval_time': 415.54486775398254, 'accumulated_logging_time': 0.02599167823791504} +I0911 15:36:35.699461 140569883633408 logging_writer.py:48] [6122] accumulated_eval_time=415.545, accumulated_logging_time=0.0259917, accumulated_submission_time=2612.91, global_step=6122, preemption_count=0, score=2612.91, test/loss=0, test/num_examples=0, test/ppl=1, total_duration=3032.49, train/loss=3.47468, train/ppl=32.2876, validation/loss=3.46714, validation/num_examples=100000, validation/ppl=32.045 +I0911 15:39:15.049693 140569892026112 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.186537, loss=3.48998 +I0911 15:39:15.052803 140605188629696 submission.py:307] 6500) loss = 3.490, grad_norm = 0.187 +I0911 15:42:44.817263 140569883633408 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.169293, loss=3.34311 +I0911 15:42:44.820509 140605188629696 submission.py:307] 7000) loss = 3.343, grad_norm = 0.169 +I0911 15:46:14.670613 140569892026112 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.163145, loss=3.4223 +I0911 15:46:14.673917 140605188629696 submission.py:307] 7500) loss = 3.422, grad_norm = 0.163 +I0911 15:49:44.464011 140569883633408 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.16491, loss=3.37403 +I0911 15:49:44.467148 140605188629696 submission.py:307] 8000) loss = 3.374, grad_norm = 0.165 +I0911 15:53:14.325482 140569892026112 logging_writer.py:48] [8500] global_step=8500, grad_norm=0.155487, loss=3.44925 +I0911 15:53:14.328588 140605188629696 submission.py:307] 8500) loss = 3.449, grad_norm = 0.155 +I0911 15:56:44.171109 140569883633408 logging_writer.py:48] [9000] global_step=9000, grad_norm=0.15259, loss=3.44755 +I0911 15:56:44.174286 140605188629696 submission.py:307] 9000) loss = 3.448, grad_norm = 0.153 +I0911 16:00:13.971422 140569892026112 logging_writer.py:48] [9500] global_step=9500, grad_norm=0.142359, loss=3.36498 +I0911 16:00:13.974629 140605188629696 submission.py:307] 9500) loss = 3.365, grad_norm = 0.142 +I0911 16:03:43.755138 140569883633408 logging_writer.py:48] [10000] global_step=10000, grad_norm=0.142133, loss=3.39824 +I0911 16:03:43.758152 140605188629696 submission.py:307] 10000) loss = 3.398, grad_norm = 0.142 +I0911 16:07:13.519424 140569892026112 logging_writer.py:48] [10500] global_step=10500, grad_norm=0.139957, loss=3.30049 +I0911 16:07:13.522597 140605188629696 submission.py:307] 10500) loss = 3.300, grad_norm = 0.140 +I0911 16:10:43.290975 140569883633408 logging_writer.py:48] [11000] global_step=11000, grad_norm=0.137574, loss=3.34706 +I0911 16:10:43.294217 140605188629696 submission.py:307] 11000) loss = 3.347, grad_norm = 0.138 +I0911 16:14:13.036739 140569892026112 logging_writer.py:48] [11500] global_step=11500, grad_norm=0.132477, loss=3.30398 +I0911 16:14:13.040705 140605188629696 submission.py:307] 11500) loss = 3.304, grad_norm = 0.132 +I0911 16:17:42.757379 140569883633408 logging_writer.py:48] [12000] global_step=12000, grad_norm=0.131727, loss=3.21332 +I0911 16:17:42.760526 140605188629696 submission.py:307] 12000) loss = 3.213, grad_norm = 0.132 +I0911 16:19:27.339489 140605188629696 spec.py:333] Evaluating on the training split. +I0911 16:19:45.095757 140605188629696 spec.py:346] Evaluating on the validation split. +I0911 16:22:36.380236 140605188629696 spec.py:363] Evaluating on the test split. +I0911 16:22:36.380950 140605188629696 submission_runner.py:516] Time since start: 5793.19s, Step: 12249, {'train/loss': 3.299115753173828, 'train/ppl': 27.08867525220346, 'validation/loss': 3.287495878956202, 'validation/ppl': 26.775729966664834, 'validation/num_examples': 100000, 'test/loss': 0.0, 'test/ppl': 1.0, 'test/num_examples': 0, 'score': 5181.371861219406, 'total_duration': 5793.191289663315, 'accumulated_submission_time': 5181.371861219406, 'accumulated_eval_time': 604.5862472057343, 'accumulated_logging_time': 0.05611538887023926} +I0911 16:22:36.403095 140569892026112 logging_writer.py:48] [12249] accumulated_eval_time=604.586, accumulated_logging_time=0.0561154, accumulated_submission_time=5181.37, global_step=12249, preemption_count=0, score=5181.37, test/loss=0, test/num_examples=0, test/ppl=1, total_duration=5793.19, train/loss=3.29912, train/ppl=27.0887, validation/loss=3.2875, validation/num_examples=100000, validation/ppl=26.7757 +I0911 16:24:22.542038 140569883633408 logging_writer.py:48] [12500] global_step=12500, grad_norm=0.12722, loss=3.32711 +I0911 16:24:22.545054 140605188629696 submission.py:307] 12500) loss = 3.327, grad_norm = 0.127 +I0911 16:27:52.140380 140569892026112 logging_writer.py:48] [13000] global_step=13000, grad_norm=0.123178, loss=3.31417 +I0911 16:27:52.143487 140605188629696 submission.py:307] 13000) loss = 3.314, grad_norm = 0.123 +I0911 16:31:21.909240 140569883633408 logging_writer.py:48] [13500] global_step=13500, grad_norm=0.118154, loss=3.29066 +I0911 16:31:21.912626 140605188629696 submission.py:307] 13500) loss = 3.291, grad_norm = 0.118 +I0911 16:34:51.710304 140569892026112 logging_writer.py:48] [14000] global_step=14000, grad_norm=0.112891, loss=3.19294 +I0911 16:34:51.713599 140605188629696 submission.py:307] 14000) loss = 3.193, grad_norm = 0.113 +I0911 16:38:21.465314 140569883633408 logging_writer.py:48] [14500] global_step=14500, grad_norm=0.115945, loss=3.27936 +I0911 16:38:21.468575 140605188629696 submission.py:307] 14500) loss = 3.279, grad_norm = 0.116 +I0911 16:41:51.256202 140569892026112 logging_writer.py:48] [15000] global_step=15000, grad_norm=0.118025, loss=3.19061 +I0911 16:41:51.259498 140605188629696 submission.py:307] 15000) loss = 3.191, grad_norm = 0.118 +I0911 16:45:20.987446 140569883633408 logging_writer.py:48] [15500] global_step=15500, grad_norm=0.109786, loss=3.15178 +I0911 16:45:20.990708 140605188629696 submission.py:307] 15500) loss = 3.152, grad_norm = 0.110 +I0911 16:48:50.662066 140569892026112 logging_writer.py:48] [16000] global_step=16000, grad_norm=0.112775, loss=3.25978 +I0911 16:48:50.665202 140605188629696 submission.py:307] 16000) loss = 3.260, grad_norm = 0.113 +I0911 16:52:20.364744 140569883633408 logging_writer.py:48] [16500] global_step=16500, grad_norm=0.110356, loss=3.24677 +I0911 16:52:20.367803 140605188629696 submission.py:307] 16500) loss = 3.247, grad_norm = 0.110 +I0911 16:55:50.073732 140569892026112 logging_writer.py:48] [17000] global_step=17000, grad_norm=0.109195, loss=3.20467 +I0911 16:55:50.076792 140605188629696 submission.py:307] 17000) loss = 3.205, grad_norm = 0.109 +I0911 16:59:19.744288 140569883633408 logging_writer.py:48] [17500] global_step=17500, grad_norm=0.110819, loss=3.17037 +I0911 16:59:19.747433 140605188629696 submission.py:307] 17500) loss = 3.170, grad_norm = 0.111 +I0911 17:02:49.378672 140569892026112 logging_writer.py:48] [18000] global_step=18000, grad_norm=0.103785, loss=3.24388 +I0911 17:02:49.381763 140605188629696 submission.py:307] 18000) loss = 3.244, grad_norm = 0.104 +I0911 17:05:27.999002 140605188629696 spec.py:333] Evaluating on the training split. +I0911 17:05:46.046527 140605188629696 spec.py:346] Evaluating on the validation split. +I0911 17:08:37.071709 140605188629696 spec.py:363] Evaluating on the test split. +I0911 17:08:37.072330 140605188629696 submission_runner.py:516] Time since start: 8553.88s, Step: 18378, {'train/loss': 3.2123374938964844, 'train/ppl': 24.837074935123002, 'validation/loss': 3.2013342191496164, 'validation/ppl': 24.565283814079528, 'validation/num_examples': 100000, 'test/loss': 0.0, 'test/ppl': 1.0, 'test/num_examples': 0, 'score': 7749.9564797878265, 'total_duration': 8553.882697105408, 'accumulated_submission_time': 7749.9564797878265, 'accumulated_eval_time': 793.6595103740692, 'accumulated_logging_time': 0.08610343933105469} +I0911 17:08:37.094881 140569883633408 logging_writer.py:48] [18378] accumulated_eval_time=793.66, accumulated_logging_time=0.0861034, accumulated_submission_time=7749.96, global_step=18378, preemption_count=0, score=7749.96, test/loss=0, test/num_examples=0, test/ppl=1, total_duration=8553.88, train/loss=3.21234, train/ppl=24.8371, validation/loss=3.20133, validation/num_examples=100000, validation/ppl=24.5653 +I0911 17:09:29.440952 140569892026112 logging_writer.py:48] [18500] global_step=18500, grad_norm=0.10458, loss=3.12959 +I0911 17:09:29.443722 140605188629696 submission.py:307] 18500) loss = 3.130, grad_norm = 0.105 +I0911 17:12:58.996543 140569883633408 logging_writer.py:48] [19000] global_step=19000, grad_norm=0.102335, loss=3.27076 +I0911 17:12:58.999632 140605188629696 submission.py:307] 19000) loss = 3.271, grad_norm = 0.102 +I0911 17:16:28.680043 140569892026112 logging_writer.py:48] [19500] global_step=19500, grad_norm=0.101115, loss=3.12139 +I0911 17:16:28.683492 140605188629696 submission.py:307] 19500) loss = 3.121, grad_norm = 0.101 +I0911 17:19:58.382122 140569883633408 logging_writer.py:48] [20000] global_step=20000, grad_norm=0.102888, loss=3.17198 +I0911 17:19:58.385255 140605188629696 submission.py:307] 20000) loss = 3.172, grad_norm = 0.103 +I0911 17:23:28.087114 140569892026112 logging_writer.py:48] [20500] global_step=20500, grad_norm=0.0984641, loss=3.18569 +I0911 17:23:28.090326 140605188629696 submission.py:307] 20500) loss = 3.186, grad_norm = 0.098 +I0911 17:26:57.782626 140569883633408 logging_writer.py:48] [21000] global_step=21000, grad_norm=0.0974808, loss=3.17962 +I0911 17:26:57.785720 140605188629696 submission.py:307] 21000) loss = 3.180, grad_norm = 0.097 +I0911 17:30:27.465466 140569892026112 logging_writer.py:48] [21500] global_step=21500, grad_norm=0.100789, loss=3.13005 +I0911 17:30:27.468548 140605188629696 submission.py:307] 21500) loss = 3.130, grad_norm = 0.101 +I0911 17:33:57.183163 140569883633408 logging_writer.py:48] [22000] global_step=22000, grad_norm=0.0973503, loss=3.11686 +I0911 17:33:57.186371 140605188629696 submission.py:307] 22000) loss = 3.117, grad_norm = 0.097 +I0911 17:37:26.886462 140569892026112 logging_writer.py:48] [22500] global_step=22500, grad_norm=0.0919339, loss=3.29487 +I0911 17:37:26.889994 140605188629696 submission.py:307] 22500) loss = 3.295, grad_norm = 0.092 +I0911 17:40:56.625951 140569883633408 logging_writer.py:48] [23000] global_step=23000, grad_norm=0.103673, loss=3.21454 +I0911 17:40:56.628957 140605188629696 submission.py:307] 23000) loss = 3.215, grad_norm = 0.104 +I0911 17:44:26.380300 140569892026112 logging_writer.py:48] [23500] global_step=23500, grad_norm=0.106521, loss=3.17386 +I0911 17:44:26.383195 140605188629696 submission.py:307] 23500) loss = 3.174, grad_norm = 0.107 +I0911 17:47:56.086103 140569883633408 logging_writer.py:48] [24000] global_step=24000, grad_norm=0.0989606, loss=3.09717 +I0911 17:47:56.089040 140605188629696 submission.py:307] 24000) loss = 3.097, grad_norm = 0.099 +I0911 17:51:25.848299 140569892026112 logging_writer.py:48] [24500] global_step=24500, grad_norm=0.0934959, loss=3.15192 +I0911 17:51:25.851377 140605188629696 submission.py:307] 24500) loss = 3.152, grad_norm = 0.093 +I0911 17:51:28.906917 140605188629696 spec.py:333] Evaluating on the training split. +I0911 17:51:46.985629 140605188629696 spec.py:346] Evaluating on the validation split. +I0911 17:54:38.332620 140605188629696 spec.py:363] Evaluating on the test split. +I0911 17:54:38.333230 140605188629696 submission_runner.py:516] Time since start: 11315.14s, Step: 24507, {'train/loss': 3.151361083984375, 'train/ppl': 23.3678485501211, 'validation/loss': 3.138734315057545, 'validation/ppl': 23.0746431402381, 'validation/num_examples': 100000, 'test/loss': 0.0, 'test/ppl': 1.0, 'test/num_examples': 0, 'score': 10318.803543329239, 'total_duration': 11315.143591880798, 'accumulated_submission_time': 10318.803543329239, 'accumulated_eval_time': 983.0857932567596, 'accumulated_logging_time': 0.11638140678405762} +I0911 17:54:38.356879 140569883633408 logging_writer.py:48] [24507] accumulated_eval_time=983.086, accumulated_logging_time=0.116381, accumulated_submission_time=10318.8, global_step=24507, preemption_count=0, score=10318.8, test/loss=0, test/num_examples=0, test/ppl=1, total_duration=11315.1, train/loss=3.15136, train/ppl=23.3678, validation/loss=3.13873, validation/num_examples=100000, validation/ppl=23.0746 +I0911 17:58:06.231453 140569892026112 logging_writer.py:48] [25000] global_step=25000, grad_norm=0.0914203, loss=3.15237 +I0911 17:58:06.234501 140605188629696 submission.py:307] 25000) loss = 3.152, grad_norm = 0.091 +I0911 18:01:35.901121 140569883633408 logging_writer.py:48] [25500] global_step=25500, grad_norm=0.0904923, loss=3.10416 +I0911 18:01:35.904031 140605188629696 submission.py:307] 25500) loss = 3.104, grad_norm = 0.090 +I0911 18:05:05.553503 140569892026112 logging_writer.py:48] [26000] global_step=26000, grad_norm=0.0886092, loss=3.0593 +I0911 18:05:05.556492 140605188629696 submission.py:307] 26000) loss = 3.059, grad_norm = 0.089 +I0911 18:08:35.272171 140569883633408 logging_writer.py:48] [26500] global_step=26500, grad_norm=0.0933994, loss=3.08532 +I0911 18:08:35.275328 140605188629696 submission.py:307] 26500) loss = 3.085, grad_norm = 0.093 +I0911 18:12:04.978625 140569892026112 logging_writer.py:48] [27000] global_step=27000, grad_norm=0.0887931, loss=3.11246 +I0911 18:12:04.981698 140605188629696 submission.py:307] 27000) loss = 3.112, grad_norm = 0.089 +I0911 18:15:34.672963 140569883633408 logging_writer.py:48] [27500] global_step=27500, grad_norm=0.0858296, loss=3.09308 +I0911 18:15:34.675965 140605188629696 submission.py:307] 27500) loss = 3.093, grad_norm = 0.086 +I0911 18:19:04.366263 140569892026112 logging_writer.py:48] [28000] global_step=28000, grad_norm=0.0876498, loss=3.11361 +I0911 18:19:04.369256 140605188629696 submission.py:307] 28000) loss = 3.114, grad_norm = 0.088 +I0911 18:22:34.022702 140569883633408 logging_writer.py:48] [28500] global_step=28500, grad_norm=0.0983615, loss=3.17031 +I0911 18:22:34.025738 140605188629696 submission.py:307] 28500) loss = 3.170, grad_norm = 0.098 +I0911 18:26:03.735054 140569892026112 logging_writer.py:48] [29000] global_step=29000, grad_norm=0.0926718, loss=3.11118 +I0911 18:26:03.738030 140605188629696 submission.py:307] 29000) loss = 3.111, grad_norm = 0.093 +I0911 18:29:33.428331 140569883633408 logging_writer.py:48] [29500] global_step=29500, grad_norm=0.0870856, loss=3.11327 +I0911 18:29:33.431256 140605188629696 submission.py:307] 29500) loss = 3.113, grad_norm = 0.087 +I0911 18:33:03.160354 140569892026112 logging_writer.py:48] [30000] global_step=30000, grad_norm=0.0890784, loss=3.08199 +I0911 18:33:03.163237 140605188629696 submission.py:307] 30000) loss = 3.082, grad_norm = 0.089 +I0911 18:36:32.856427 140569883633408 logging_writer.py:48] [30500] global_step=30500, grad_norm=0.0944396, loss=3.10263 +I0911 18:36:32.859644 140605188629696 submission.py:307] 30500) loss = 3.103, grad_norm = 0.094 +I0911 18:37:30.013945 140605188629696 spec.py:333] Evaluating on the training split. +I0911 18:37:47.753752 140605188629696 spec.py:346] Evaluating on the validation split. +I0911 18:40:38.642781 140605188629696 spec.py:363] Evaluating on the test split. +I0911 18:40:38.643370 140605188629696 submission_runner.py:516] Time since start: 14075.45s, Step: 30636, {'train/loss': 3.1036386489868164, 'train/ppl': 22.278868960619352, 'validation/loss': 3.0924446905970266, 'validation/ppl': 22.03087085832684, 'validation/num_examples': 100000, 'test/loss': 0.0, 'test/ppl': 1.0, 'test/num_examples': 0, 'score': 12887.495035886765, 'total_duration': 14075.453748703003, 'accumulated_submission_time': 12887.495035886765, 'accumulated_eval_time': 1171.715161561966, 'accumulated_logging_time': 0.1475672721862793} +I0911 18:40:38.665940 140569892026112 logging_writer.py:48] [30636] accumulated_eval_time=1171.72, accumulated_logging_time=0.147567, accumulated_submission_time=12887.5, global_step=30636, preemption_count=0, score=12887.5, test/loss=0, test/num_examples=0, test/ppl=1, total_duration=14075.5, train/loss=3.10364, train/ppl=22.2789, validation/loss=3.09244, validation/num_examples=100000, validation/ppl=22.0309 +I0911 18:40:39.195161 140569883633408 logging_writer.py:48] [30636] global_step=30636, preemption_count=0, score=12887.5 +I0911 18:40:39.203177 140605188629696 submission_runner.py:857] Final finewebedu_lm score: 12887.495035886765 diff --git a/logs/self_tuning/ademamix_golden/study_1/finewebedu_lm_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_1/finewebedu_lm_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..ccc4c538 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/finewebedu_lm_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,7 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/loss,test/num_examples,test/ppl,total_duration,train/loss,train/ppl,validation/loss,validation/num_examples,validation/ppl +226.6703073978424,0.0,44.63611102104187,1,0,44.63611102104187,0.0,0,1.0,271.8861677646637,11.020582580566408,61119.27614337982,11.021556805466751,100000,61178.84907809615 +415.54486775398254,0.025991678237915,2612.907131433487,6122,0,2612.907131433487,0.0,0,1.0,3032.4875638484955,3.4746829986572267,32.28759170422644,3.46714123191736,100000,32.04500214577074 +604.5862472057343,0.0561153888702392,5181.371861219406,12249,0,5181.371861219406,0.0,0,1.0,5793.191289663315,3.299115753173828,27.08867525220346,3.287495878956202,100000,26.77572996666484 +793.6595103740692,0.0861034393310546,7749.956479787826,18378,0,7749.956479787826,0.0,0,1.0,8553.882697105408,3.2123374938964844,24.837074935123,3.2013342191496164,100000,24.565283814079528 +983.0857932567596,0.1163814067840576,10318.80354332924,24507,0,10318.80354332924,0.0,0,1.0,11315.143591880798,3.151361083984375,23.3678485501211,3.138734315057545,100000,23.0746431402381 +1171.715161561966,0.1475672721862793,12887.495035886765,30636,0,12887.495035886765,0.0,0,1.0,14075.453748703003,3.1036386489868164,22.278868960619352,3.0924446905970266,100000,22.03087085832684 diff --git a/logs/self_tuning/ademamix_golden/study_1/finewebedu_lm_pytorch/trial_1/events.out.tfevents.1789137963.1aa19998bf4d.38.0.v2 b/logs/self_tuning/ademamix_golden/study_1/finewebedu_lm_pytorch/trial_1/events.out.tfevents.1789137963.1aa19998bf4d.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..35edb4b6633952ebf1451e054214bb51f54020f3 GIT binary patch literal 39308 zcmbW=cR)|;9|!P|(N|U?Qp&23WJF0cRFs*hgq9*&8ukd;Aqm+TnGsP|lo2H>vb)(t zQFaKw+xO%-=Y5{%xcS|G@BO^*^E%Hs-_LiPdMR-K+`D1^Qb|D}T~>Y1eGBzEp1~fz z0S4BSyaPgebarShYZ>Yt5E2;d;}Fzd#R(_Jb8ly_+lSGt^B z5uhj2k<}jS7wGBX=N1y`9rQb*{6B^W#ieT^x>iJt?xHqQ>5$rmf#u_`U!|R{tc^?- zT7FGB3LOI-MUK3kJ5yj7_gat)xrECI3iA6s4BNB-TNuXndkue?R19Nj&F=4%sh6*&rf&hAsvG$A=%Wu0Yef}DUbe>d+b9{xdo-ej^B2A@OWj5-en!$q<*CxP5W z@(u}w>s=_C){IcXG<7K zz`U2cV4ga;l5+Vxg$9QHyLQKf1cTT{&g;tQ0l4NUWT``Qp7vl~pt@9u`#ONC#PaS^c13Z*Tvg@;&F~6&Mx} z`g;K>khJz{&5$YKHY$n>WZlybiz|PQFN~+lh;ZN~FsL_1R zd$=A8qDe%huSec(>n)(LDeTWFs?vvfEe(<4&ij3%L6P^KRI?S-ajw;1;a*T&_(me6 z6$<9CG<<#~MSx;Z(l}00jaJyicEnqf!7a6TP+UAtsw=5@)@`yEC=Q(=5z-2CR1E46 z^#T;G?h%|qkygYM7@(qPr{Pmj)E!5vE2+pXwzdOBgCG(itq4a&yBQB+LGfM|$0@4Q zik;UwpyJw?CbdEFLzPrlQnAUgM-nKmYLEzNMI0(-=&nryMZnIJoT3J;2#?iA#o{U9 zjX_cEAgQjTqPAAzDNrmsMk1sY>8LR3utXaa`W|;UMNL}qvqyVWI9>Rq3W_`9NHtqQ zol*BvFK+~ee-Md~Ry;yQY%9-ApqN?h1E;7(D<=Na78MD7t9=ErvYhO4hf(NzZ6f#2gT$sq`H!dfLmJ*fx_b#iI7%^?!Kcoyx9YaFEJjRLYYI?@N#dq=Uj=IwAC@8A7B@xn!C>+O_f%9sB zqU>x-PSKE7n6^~I>XNT-@g1uc*`&IX3RC4-E}(dRgG5LxVsRY9EYc=|;(%8#PN7OG zHr{EB)!BthKf_%7m_Vx8if`h%aE$Gc1B!zoBtlvdhvSG_wzn7*PS;&HMI%~K79diT zMeo4ZwEgaq>PjlQj%$eb?uQRagtTHeDgrjmbpyrj;bEMjF|7zz5Gn5ZRht5H;X0aB zvlZ07I5fL|Gnk9J-Xua=aS#>J$rD^a(f3;vr%bpsGB^A+LhdYAe zU?Pc-R;1%Nb`^YF2a0v=GdV?5T48JgCg3LL`W+# zQK30@>BF;Fo|^rR@xgt zku&WVr)Wki3?oH~s_L&}L18?XRI?S-TwF_4#TE5nF^P~?+(kvP#@QX97+1ZSgZ$pD zK`T@yixeYn4Q~YshdQL1t)O7JwrgDndQ$Zv}lFOIguhPC;S2^R_2guw&IKUjH=x= z{R1du_eg}aq7)T7lZ^~Ov3}%YPN7XJV)R9d*2+$rpcp-dRI?S->!`%5dapt8V*-hg zRw&?yQS&@C)_`K0ES^)epcRj^8sIg^TAtk#6y_?VnysMbVu$&SR-hu(jQq5LSE9%b7`g=fOX+$Ra3E_+0C;o|-T z`|hj$q?)bxEIy+GbAu0p;&m8_kXERoqEU-hcA$7sXuv6SXhr8^Hz;x@lj=$;-n!So z&o8deCK1vK4OC43V7wm`4eR=F3O!nJAzh>}Kd$!#6h)0lHCsWQwC_%hjfNF9OOr%M zE3{G3&10H6C|1PJ;S_CYMVlESMX0*77bx28C)I2PwW4$$dSTx^C7DD>D_WyM*Uxqi zD28;}#3|a*ikNC5h1une??ItpLaNyc>WsR2_RAWW3p)!EA+69u#j}X~j-Ys$cZ^fC zrxk5I>)`Ibet3TiP;7Wcsw=6Oe@mqSC>9ry2x)~rDmF!{;qK0vn#(ElX+_rvk;2PQ zI|mejb4WE?LG6nj5wYt)(SI?CkX9I?Vu97lwlEjgCB>Yg1Fbmkq>OX1_KxaNP^g!Z zYPN#f7x95zhJvDzLdov|NGrOcqR&qcUr^MXTH8^6*E66M2fRg!0i!K;g5p&;sb(vv zS0J-&_YVTa>_sF(T49WeX_3$Iu2;8_E~n^7E1oP9DLyAX90iKG8l;-7pmw)Q(^wNw z^lD8aq!m3;(R!H`zBd?j$dXeS(uxcJ$Z$o4_zYeE3a{g&x{`|Lbw8{Gg~}NcA+0b+ zMe^7c@vx#StwwW-P7K9Uk>Wz+^e9klvM1GS1+}8|o-I!X#loQ^LR!%W6-_rPXo8~T z!BkGsnN~FKTpQmG2F1C0hd4!7T5;M~q!`^t z^odwsHB!x1P`f+wbrhboh0RHXv|=zSPR#F}4~pEmmpO$Itx)`>ge&TjdCCb`QS+9N zYPN!!i*Gqg?O`sOMw19>g(E5újMLpdDPSK54w9*$TGW^%#?#?qH)ocZ|FT8E< znS)|QHxePOa7IN=!%nY3vF>)YVe-3PcUqw{RHSelG0GI?qDnrgW-F)_b*^;KE>N6$ zP9mfgBT><5jn8dRZ1vIN6vhlimPnCh-}4hF`UH?_wt||A`>Q(j21WfzBtlx@j*1ya zHr@fnx_YLZ!h}`~4y%RxVr0f-Lr_#vBh_pLH5Vg__v5!&vYL|!X~h^++>cV81&VHm zhI5J@wBl5Ck)qy_?e?I^J4UK2sc`6dF&*Y&V>*eDR*XZ%j0ty7gW^G_U`}C5D}uU< z6vNtoQ2|A7Q&L??h340F3qfILMIxjXeyEuE*|i}kF6>&yDSFZhhp8e(!>MnbK`}Lv zRI?S-Njp5Zp${m2rjQ6}MGz`pTrI{|gMJo!IE5LlXw|JI?u(;)=05{PT{}|ER(uk_ zA9eN8P%TiD4j~cJicnOHJ~IC(%*DpX=Q%|$TCu#bNU`6}#sCyXg`}FTpmul8$+UBz z2>(DLq!m+8p*4S0YfzjDe#9wy(~9VRHE=H4b=~w96t2@qbtM(eYHQ)X_&SF~NGoQb z;#py{xu8(1U&blSX~oF3B88dTvsIuttwyTZ3Tj_$=zav>8?@3Q5z>lqRHPY9hz3RH z18T$N_io0|(0n3Aim8oMu7Tp{QBuuTPfU`|ME(#^XbvM0(uxRF?75na-}3r(c_62-q7_fp zh!oWa_~K9L8r~t*Yz6hHeO7)7epf9ypF~J27NBC=cU8P!d>=8MQ}m$~eR4#KW)9WX zz+5DGl4`bs+7})^rww2(V#bpQX~iN`glQ)Hgca4XY(A&3rWN=0RmT;T)J$6$6i*Zv z{H|szsP7GqJojb`C>|@52x-Lm~s95pc6W={(lyR3+^raOinu-)&89p;%MSZ_as@V!^U+lKcR074`ToNIz zh(blj+D-bxiW)uUBd6#`E7nvMDeR(kqd`%}k5sc2)QYOVe#&@IJP9EY(u!zQyjH%{R zzOFqNVoxiaFI2-bs>j1x0ifu)om8_G)V({h>6W`N7snDvgtTG}Dh`!JtAL_z16xkf zpH}SoS_KvAr{~TGMZ2b?nysLAcZR~&cA&VWO(LWfv8Y&k-`WHe2jVz+%BtlxT9u*J7`s>du&F_2cIo)pbRe&Y@cL6K%f zs@V$a709t4=gxxSnmvh-R%}Fts>=Oapip?aj#CVx6)x{Yih)Dw;b*KfUz6%eDh?Rw zJp)D3dlDh7*o=ydGiD<|kr0~9DF)Muu|^8Gq8wgsy90CKHiJ~N71T-FYqjlhP&}GP zBBT{@sJOCgsWK>Te!k8rhR_OiE0JPsv-Lkgk^GZXvlY}CHAs8!Nl@6=$oV}0X~i~F z7%eer0E$gpi?|R6TJf-+NOA6FbuCbo?IG1{1$Do$bavAO#hODTLRzr{6%#uboCigC zhg#0^yWUV*QR*O4Om6?V3=}PmNOdI@^@`N@fc52NOw*zP)sQ#jFz$eyCPaMhSG3FhK}7pZ0|sFPOx z!^HZqqJ~W%5z>kSsL=E3xfB%LpRVK-&a`4)JJDQZ>J4oK3hhEt%~nvaqaL1_`vVkd z?@5HT;vg#YcWuY_K)hz`=M*lqqQ;i*xT2aWO_~mhf_bExt)Nzv^4ODj(v~hI5z>kz zRP=TaX$*6*K|706jGz_Mmx&a|vW$H}VW&r`*$V30k3)Za?F@5aY)B%c6-Q99`HARx zAa!;>;}ov6VrVmwB5;SE0w{hZl4`bsI-^G1Y7_{H>qkk1v?2u+A4ZmSfw{2KQgD&q z^+wW)al^jhiYn4c>)^Lp8eAaNYy~wJw{~Qv!CWL>AraDw6R2=7F6#qxQ5@2pQ;en+EnkWh2T$AK zuMW>oC)I2PH5a!gWuPKv9*K}vq@to=%kFES$gMk+Q@GKJCC;Td7eD)L=>;pQxf-di zq@uV+>@Ap!d@T|otvHQ}yDQ9eK@qbffK#~Bit*`RP!WAx58vUOnn0>6skl?Wt0ySD zj*tjxMLH_>w04dFMXuEnPT@f-z88H)#gBdcB4I8T*^_Fvg1YN@bUlmr?zuxrgtX!; zDn6PW?hcB{dGVaWlU8gUEmC|QH3WaT5cZ5zvlZ0tPMm*nEzCu)A`&63IFE`0vbENr zC>(l*Q+Uye{o$W*E;c;f)E?$y<|tClR#3aUU(l2{pxEm}BBT`;P|>7$DBinc^71&v z7+Rt8P_(-XHWcITE`3U>*$V2_;N0{*`1!@rLJ}dZ$V5fV3cE(|jJ>KhDf8ray^oBC zd6mse!pS(Yu8A%@+c9_$U(R+Ei$C$MW&U(Ww~9x*4RVSsR$A-CXGlZ7;S=y2UApB0 zO8Zxa9JYCIms*?#o zocFCNl0)(skIU(x5P^`>SxTJ&^8N^4&W6}uzlab0Y%^!eDuBF6+HGB3jIpjR!ffVkk6A|Ludy@x14&36)*^tL~E+8b|Kl3?27XHI$NJ7jttYK-L z-8J0;AaOswE|x=%F`gjssE!{(d~&^#0HRv+8!2Z)7ALMm$dIyMD*@8F9-koz2{%PZ z^+PWD09mdzeFQ>I zcE~LTNPU+N@8ys<#&2{7TY4j;b5=cdfc*5}%h`}$`I8XRu2IW_06FW+XGlU6rXj>` z?Do$9(Z1=YTeEz#$1r|Wx!~Lwgf!`MAp;d!4v>rXuU+MkMT}p;eX;XG$h6S1V1SGq#+S1po;kx1Vy#oFJ3wBK zf(d^8xWU6Bg<9o%Ct(|Th&Z|F;hMmiS-^+LIn<-DXX#YI7k%57ekp{<@;Vgr z8Gm{f_KxSH@Z!#GbM6^DRNLn>pY(@^VK*{Rz#(;+{ALI|z zZ4}>-tIb`g|9l*FX38q;HOypuM&+J|y`azi)SeZuZZnT=cTlouz(@S$1F@TRA9via zB5U=h>uyRdoA6om6R{%(w!#J2FLlsMn62H-`T9RR68p+RGYm>L+@4`mF_2)R{XRm5O&wSd zAeU?ITPcTJX8b7OXAM_`)VCNk0U-Sv@a1esa6Mgwq)mv~2{Yo+l+TcaxW7ioao2~B z0kY+a<^?$x-{g=ZjIaGJNFRw1U#FjA0kW}_FK0vQ7;7P9Mf`_wfY>XnCPR>fTvxP#)5qan z!D)artXsRIQuzyr-Hh*Pp4cIRv~;{Q1|Uz>_;NNR)La80pL60q0;H`rpCJi}Q$|SC z*fll)aj>1`C5LQee6jAXt7vKU)oQ2;kdK4;ayDe{g$4-8o^>9V)@x@zLlP3CiV&l- zd6obMJRc7;qCbnzkc60PAmqv; zoi_kk;^J{x4vAoVb)~K8P=wS-v%`*hnq__#Ifv@sHYAH46ruN#>!ThK}Rs@C!h3({w=BNZTuc#$K@}zpnp47U7cK;|Eqs9w;A6N=KWP| zK9Rdn|JlDyDfP^P&x(vy_>9W+ZzpqB%!2-{s0knOm;Kwll#(&%)2}r_(7);F^FfvE->Bgo3%=JG`nUdF_|*UI-{!Yb z!}r}2vt~?#{%uJQKCe>!oB2I&Gk68{V087`a1gw<;3Fjuf-$}EAXu)m!!*Oobwkz^8C71ER^uS_A+`q2sTDxGAG`zu=vk%Ab)>ZLv{Ji`@H@GHM z?(-Rv*JQ92UXyY9Wm^GK(nzUl?ehC>rZ@e0==f$LNL0huy8$v)i!WzGs#=%fZN2D3)v%me10c%Id^sC(;@dlfOjfUHaqFvaCJpBtVXC;LF*Nz%DNla`x!* zJbs4=_3%*YPo+ufb@PgYKt7QV6 zb`>B$-S`Yih+!~7>WsJC0g(CA_V$!R!e`QuzHx&QV%GW7OMv9fJy~w z4v4SQz2Xhmmj#zxlp6?F`1HtM=y@q8et zXInOIeTWvdg^ve-Me(U1MAC9S0xcKK;{CzmbGrDR+>*D5VHy0=4(H|6B2_i8EVwAB zXIn}?isnUAF>60q{(Z^?KL|<7u|;Sxn7!cuSXK>pu3M*k*I$ohSPpIwt+8d9&ojWX z*iBH+wwNr-#d)ze%<2!8q_Kh!NlRQLS{B$BTc@Fu@(;nQ$(d$?kUzC^d!C5y5Tzl7pP@0d=p%^wLqB(9{&^f! zs!_2Py!IYqFG%{o8oJV?RQwwlvxW#Fxr_Cm9o_SWsZMa>CO8Q~D%aAb=_DJPRUA|9 z{oWNRnT!@>{AEu!KVN?>{DtMTF@i9O z87x0y&POqh%MSK^@#udt@oXJ9TndK^>e&OA=rDYqPpVOVq=MX$kC3tu$>i$x#}P~$K0N!_k?-b_C`?8w)7i+6)iK<+SCL~jZcCQNlWQ2wA6nQ zKLIQzx9lIwEhYOHmYvf?dw+WTa|f_o$QRVJEzxeGd6_xIBm*onUkE}ZEsyr0<l-X99U>Px65F%-bJ&G1n zi&^e4FGd@xAC_CvPckeujBW7#@+>*>2w1A`5Y)3RclTXFOWOK#dSIEhPY@z$2|tOJ z4$b;K2TPv!q?dBbku-*-?3n0XtF1FCAzV}c5Ytfapk;%Bm(=RY2#8OtPii$!@#TSVpQQ7i+UqR&aO^_lN zQSGbiyA#+c9_{tZAKKS13TyeH{HcAd@mO@H@V~!xCbU9RYYL+N(7ry575$OH<5j$~ zp?zIeSCI67wXd0L9jxJEjaC&zau@4A+t&sWroEtj?XNBfsa*Ryw~mbrVyw=vUxlfm-j zvY?)Asp)teE%uQo-N3ReR}dm;F}{cv!;6Qq!E$?e-gmOQ&#N8c%5P6D$wA3PL0;QBTm~X*FI6=H<;?*H$X!dq4FB!*aH_$f6&A zy+6#$!liBf$s@)NZy+zw6`QmapiteF(IC)z1q;{?zhqdM#QlMJ1Nf7mimd~X3 zK^#;>wGuUG`4X=SlK!ujZ*`F+{>8diw*`^h#rn^dukiRP3uyTw^8_K4YxySVx%7dS zuhvsR#$UF4LpJ%uLCe?Vl^~36`M!_phbuAT(7MwwXWGSrfIqZ+Id5DsX~mLl2GH{T z_$WxKY|BRt@M43HmeBH5`7Vh4@0RcBq-bpU<_zxJ16sZ>Rrc|FuTm}Fqh@OOwafeG zgU7+Ku&&1 | tee -a /logs/imagenet_resnet_pytorch_09-12-2026-11-08-18.log +W0912 11:08:21.756000 9 site-packages/torch/distributed/run.py:803] +W0912 11:08:21.756000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0912 11:08:21.756000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0912 11:08:21.756000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-12 11:08:30.525326: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-12 11:08:30.525326: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-12 11:08:30.525325: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-12 11:08:30.525327: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789211310.552408 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789211310.552407 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789211310.552408 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789211310.552417 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789211310.566856 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789211310.566856 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789211310.566858 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789211310.566875 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789211310.624657 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789211310.624657 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789211310.624657 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789211310.624657 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789211310.624688 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789211310.624688 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789211310.624688 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789211310.624690 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789211310.624691 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789211310.624691 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789211310.624691 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789211310.624692 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789211310.624693 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789211310.624693 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789211310.624694 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789211310.624696 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789211333.565215 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789211333.567302 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789211333.579603 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789211333.641822 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W912 11:09:08.243851434 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0912 11:09:09.341496 140000775775424 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/imagenet_resnet_pytorch. +I0912 11:09:09.341499 140058249217216 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/imagenet_resnet_pytorch. +I0912 11:09:09.341495 140225541866688 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/imagenet_resnet_pytorch. +I0912 11:09:09.341510 140599000147136 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/imagenet_resnet_pytorch. +I0912 11:09:09.366657 140000775775424 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/imagenet_resnet_pytorch/trial_1. +I0912 11:09:09.674688 140000775775424 submission_runner.py:242] Initializing dataset. +I0912 11:09:30.042046 140000775775424 submission_runner.py:251] Initializing model. +I0912 11:09:30.613112 140000775775424 submission_runner.py:290] Performing `torch.compile`. +I0912 11:09:34.420109 140000775775424 submission_runner.py:294] Initializing optimizer. +I0912 11:09:34.420954 140000775775424 submission_runner.py:299] Initializing metrics bundle. +I0912 11:09:34.421108 140000775775424 submission_runner.py:321] Initializing checkpoint and logger. +I0912 11:09:34.423401 140000775775424 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_1/imagenet_resnet_pytorch/trial_1/meta_data_0.json. +I0912 11:09:34.782319 140000775775424 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_1/imagenet_resnet_pytorch/trial_1/flags_0.json. +I0912 11:09:34.822401 140000775775424 submission_runner.py:359] Starting training loop. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +[rank2]:W0912 11:10:01.099000 40 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank3]:W0912 11:10:01.317000 41 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank0]:W0912 11:10:01.930000 38 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank1]:W0912 11:10:02.282000 39 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +/usr/local/lib/python3.11/site-packages/torch/autograd/graph.py:841: UserWarning: Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 512, 512] +bucket_view.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 1, 1] (Triggered internally at /pytorch/torch/csrc/distributed/c10d/reducer.cpp:334.) + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +/usr/local/lib/python3.11/site-packages/torch/autograd/graph.py:841: UserWarning: Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 512, 512] +bucket_view.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 1, 1] (Triggered internally at /pytorch/torch/csrc/distributed/c10d/reducer.cpp:334.) + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +/usr/local/lib/python3.11/site-packages/torch/autograd/graph.py:841: UserWarning: Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 512, 512] +bucket_view.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 1, 1] (Triggered internally at /pytorch/torch/csrc/distributed/c10d/reducer.cpp:334.) + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +/usr/local/lib/python3.11/site-packages/torch/autograd/graph.py:841: UserWarning: Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 512, 512] +bucket_view.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 1, 1] (Triggered internally at /pytorch/torch/csrc/distributed/c10d/reducer.cpp:334.) + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +I0912 11:12:38.901675 139981372782336 logging_writer.py:48] [0] global_step=0, grad_norm=0.499999, loss=6.92365 +I0912 11:12:39.143537 140000775775424 submission.py:307] 0) loss = 6.924, grad_norm = 0.500 +I0912 11:12:39.811351 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +I0912 11:14:01.136288 140000775775424 spec.py:346] Evaluating on the validation split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 11:15:38.706774 140000775775424 spec.py:363] Evaluating on the test split. +I0912 11:15:39.791303 140000775775424 dataset_info.py:707] Load dataset info from /data/imagenet/pytorch/imagenet_v2/matched-frequency/3.0.0 +I0912 11:15:39.801545 140000775775424 reader.py:262] Creating a tf.data.Dataset reading 16 files located in folders: /data/imagenet/pytorch/imagenet_v2/matched-frequency/3.0.0. +I0912 11:15:39.898643 140000775775424 logging_logger.py:49] Constructing tf.data.Dataset imagenet_v2 for split test, from /data/imagenet/pytorch/imagenet_v2/matched-frequency/3.0.0 +I0912 11:16:11.048904 140000775775424 submission_runner.py:516] Time since start: 396.23s, Step: 1, {'train/accuracy': 0.001136001275510204, 'train/loss': 6.911248654735331, 'validation/accuracy': 0.00026, 'validation/loss': 6.914865, 'validation/num_examples': 50000, 'test/accuracy': 0.0008, 'test/loss': 6.91190625, 'test/num_examples': 10000, 'score': 184.32222723960876, 'total_duration': 396.2264976501465, 'accumulated_submission_time': 184.32222723960876, 'accumulated_eval_time': 211.23752093315125, 'accumulated_logging_time': 0} +I0912 11:16:11.307137 139942265014016 logging_writer.py:48] [1] accumulated_eval_time=211.238, accumulated_logging_time=0, accumulated_submission_time=184.322, global_step=1, preemption_count=0, score=184.322, test/accuracy=0.0008, test/loss=6.91191, test/num_examples=10000, total_duration=396.226, train/accuracy=0.001136, train/loss=6.91125, validation/accuracy=0.00026, validation/loss=6.91486, validation/num_examples=50000 +I0912 11:16:12.807030 139942256621312 logging_writer.py:48] [1] global_step=1, grad_norm=0.499999, loss=6.92678 +I0912 11:16:12.810521 140000775775424 submission.py:307] 1) loss = 6.927, grad_norm = 0.500 +I0912 11:16:13.084067 139942265014016 logging_writer.py:48] [2] global_step=2, grad_norm=0.499999, loss=6.93172 +I0912 11:16:13.089263 140000775775424 submission.py:307] 2) loss = 6.932, grad_norm = 0.500 +I0912 11:16:13.366216 139942256621312 logging_writer.py:48] [3] global_step=3, grad_norm=0.499999, loss=6.9365 +I0912 11:16:13.370569 140000775775424 submission.py:307] 3) loss = 6.936, grad_norm = 0.500 +I0912 11:16:13.647101 139942265014016 logging_writer.py:48] [4] global_step=4, grad_norm=0.499999, loss=6.92616 +I0912 11:16:13.650180 140000775775424 submission.py:307] 4) loss = 6.926, grad_norm = 0.500 +I0912 11:16:13.926536 139942256621312 logging_writer.py:48] [5] global_step=5, grad_norm=0.499999, loss=6.92195 +I0912 11:16:13.929568 140000775775424 submission.py:307] 5) loss = 6.922, grad_norm = 0.500 +I0912 11:16:14.247255 139942265014016 logging_writer.py:48] [6] global_step=6, grad_norm=0.499999, loss=6.92696 +I0912 11:16:14.250463 140000775775424 submission.py:307] 6) loss = 6.927, grad_norm = 0.500 +I0912 11:16:14.526139 139942256621312 logging_writer.py:48] [7] global_step=7, grad_norm=0.499999, loss=6.92254 +I0912 11:16:14.531712 140000775775424 submission.py:307] 7) loss = 6.923, grad_norm = 0.500 +I0912 11:16:14.805750 139942265014016 logging_writer.py:48] [8] global_step=8, grad_norm=0.499999, loss=6.92882 +I0912 11:16:14.810229 140000775775424 submission.py:307] 8) loss = 6.929, grad_norm = 0.500 +I0912 11:16:15.077357 139942256621312 logging_writer.py:48] [9] global_step=9, grad_norm=0.499999, loss=6.92124 +I0912 11:16:15.081458 140000775775424 submission.py:307] 9) loss = 6.921, grad_norm = 0.500 +I0912 11:16:15.358593 139942265014016 logging_writer.py:48] [10] global_step=10, grad_norm=0.499999, loss=6.9284 +I0912 11:16:15.362551 140000775775424 submission.py:307] 10) loss = 6.928, grad_norm = 0.500 +I0912 11:16:15.657039 139942256621312 logging_writer.py:48] [11] global_step=11, grad_norm=0.499999, loss=6.92496 +I0912 11:16:15.660885 140000775775424 submission.py:307] 11) loss = 6.925, grad_norm = 0.500 +I0912 11:16:15.934102 139942265014016 logging_writer.py:48] [12] global_step=12, grad_norm=0.499999, loss=6.91983 +I0912 11:16:15.937209 140000775775424 submission.py:307] 12) loss = 6.920, grad_norm = 0.500 +I0912 11:16:16.232520 139942256621312 logging_writer.py:48] [13] global_step=13, grad_norm=0.499999, loss=6.92718 +I0912 11:16:16.236246 140000775775424 submission.py:307] 13) loss = 6.927, grad_norm = 0.500 +I0912 11:16:16.508719 139942265014016 logging_writer.py:48] [14] global_step=14, grad_norm=0.499999, loss=6.9309 +I0912 11:16:16.513473 140000775775424 submission.py:307] 14) loss = 6.931, grad_norm = 0.500 +I0912 11:16:16.781800 139942256621312 logging_writer.py:48] [15] global_step=15, grad_norm=0.499999, loss=6.92414 +I0912 11:16:16.785180 140000775775424 submission.py:307] 15) loss = 6.924, grad_norm = 0.500 +I0912 11:16:17.086954 139942265014016 logging_writer.py:48] [16] global_step=16, grad_norm=0.499999, loss=6.92805 +I0912 11:16:17.091365 140000775775424 submission.py:307] 16) loss = 6.928, grad_norm = 0.500 +I0912 11:16:17.369531 139942256621312 logging_writer.py:48] [17] global_step=17, grad_norm=0.499999, loss=6.92559 +I0912 11:16:17.372584 140000775775424 submission.py:307] 17) loss = 6.926, grad_norm = 0.500 +I0912 11:16:17.654225 139942265014016 logging_writer.py:48] [18] global_step=18, grad_norm=0.499999, loss=6.92554 +I0912 11:16:17.657337 140000775775424 submission.py:307] 18) loss = 6.926, grad_norm = 0.500 +I0912 11:16:17.933720 139942256621312 logging_writer.py:48] [19] global_step=19, grad_norm=0.499999, loss=6.92525 +I0912 11:16:17.936938 140000775775424 submission.py:307] 19) loss = 6.925, grad_norm = 0.500 +I0912 11:16:18.213970 139942265014016 logging_writer.py:48] [20] global_step=20, grad_norm=0.499999, loss=6.91782 +I0912 11:16:18.219237 140000775775424 submission.py:307] 20) loss = 6.918, grad_norm = 0.500 +I0912 11:16:18.506658 139942256621312 logging_writer.py:48] [21] global_step=21, grad_norm=0.499999, loss=6.92134 +I0912 11:16:18.510620 140000775775424 submission.py:307] 21) loss = 6.921, grad_norm = 0.500 +I0912 11:16:18.783497 139942265014016 logging_writer.py:48] [22] global_step=22, grad_norm=0.499999, loss=6.91442 +I0912 11:16:18.787189 140000775775424 submission.py:307] 22) loss = 6.914, grad_norm = 0.500 +I0912 11:16:19.058640 139942256621312 logging_writer.py:48] [23] global_step=23, grad_norm=0.499999, loss=6.93052 +I0912 11:16:19.062879 140000775775424 submission.py:307] 23) loss = 6.931, grad_norm = 0.500 +I0912 11:16:19.340230 139942265014016 logging_writer.py:48] [24] global_step=24, grad_norm=0.499999, loss=6.93953 +I0912 11:16:19.344379 140000775775424 submission.py:307] 24) loss = 6.940, grad_norm = 0.500 +I0912 11:16:19.608510 139942256621312 logging_writer.py:48] [25] global_step=25, grad_norm=0.499999, loss=6.92371 +I0912 11:16:19.612462 140000775775424 submission.py:307] 25) loss = 6.924, grad_norm = 0.500 +I0912 11:16:19.912442 139942265014016 logging_writer.py:48] [26] global_step=26, grad_norm=0.499999, loss=6.92816 +I0912 11:16:19.917130 140000775775424 submission.py:307] 26) loss = 6.928, grad_norm = 0.500 +I0912 11:16:20.186724 139942256621312 logging_writer.py:48] [27] global_step=27, grad_norm=0.499999, loss=6.92449 +I0912 11:16:20.189734 140000775775424 submission.py:307] 27) loss = 6.924, grad_norm = 0.500 +I0912 11:16:20.461868 139942265014016 logging_writer.py:48] [28] global_step=28, grad_norm=0.499999, loss=6.92281 +I0912 11:16:20.465828 140000775775424 submission.py:307] 28) loss = 6.923, grad_norm = 0.500 +I0912 11:16:20.753783 139942256621312 logging_writer.py:48] [29] global_step=29, grad_norm=0.499999, loss=6.91998 +I0912 11:16:20.758304 140000775775424 submission.py:307] 29) loss = 6.920, grad_norm = 0.500 +I0912 11:16:21.046257 139942265014016 logging_writer.py:48] [30] global_step=30, grad_norm=0.499999, loss=6.92918 +I0912 11:16:21.050020 140000775775424 submission.py:307] 30) loss = 6.929, grad_norm = 0.500 +I0912 11:16:21.355234 139942256621312 logging_writer.py:48] [31] global_step=31, grad_norm=0.499999, loss=6.91952 +I0912 11:16:21.358339 140000775775424 submission.py:307] 31) loss = 6.920, grad_norm = 0.500 +I0912 11:16:21.636452 139942265014016 logging_writer.py:48] [32] global_step=32, grad_norm=0.499999, loss=6.91052 +I0912 11:16:21.639483 140000775775424 submission.py:307] 32) loss = 6.911, grad_norm = 0.500 +I0912 11:16:21.919319 139942256621312 logging_writer.py:48] [33] global_step=33, grad_norm=0.499999, loss=6.92861 +I0912 11:16:21.923393 140000775775424 submission.py:307] 33) loss = 6.929, grad_norm = 0.500 +I0912 11:16:22.202774 139942265014016 logging_writer.py:48] [34] global_step=34, grad_norm=0.499999, loss=6.91866 +I0912 11:16:22.206728 140000775775424 submission.py:307] 34) loss = 6.919, grad_norm = 0.500 +I0912 11:16:22.482451 139942256621312 logging_writer.py:48] [35] global_step=35, grad_norm=0.499999, loss=6.91346 +I0912 11:16:22.487232 140000775775424 submission.py:307] 35) loss = 6.913, grad_norm = 0.500 +I0912 11:16:22.766621 139942265014016 logging_writer.py:48] [36] global_step=36, grad_norm=0.499999, loss=6.92678 +I0912 11:16:22.771980 140000775775424 submission.py:307] 36) loss = 6.927, grad_norm = 0.500 +I0912 11:16:23.050116 139942256621312 logging_writer.py:48] [37] global_step=37, grad_norm=0.499999, loss=6.91432 +I0912 11:16:23.055119 140000775775424 submission.py:307] 37) loss = 6.914, grad_norm = 0.500 +I0912 11:16:23.331223 139942265014016 logging_writer.py:48] [38] global_step=38, grad_norm=0.499999, loss=6.92983 +I0912 11:16:23.335253 140000775775424 submission.py:307] 38) loss = 6.930, grad_norm = 0.500 +I0912 11:16:23.602872 139942256621312 logging_writer.py:48] [39] global_step=39, grad_norm=0.499999, loss=6.92583 +I0912 11:16:23.606024 140000775775424 submission.py:307] 39) loss = 6.926, grad_norm = 0.500 +I0912 11:16:23.929395 139942265014016 logging_writer.py:48] [40] global_step=40, grad_norm=0.499999, loss=6.91779 +I0912 11:16:23.934859 140000775775424 submission.py:307] 40) loss = 6.918, grad_norm = 0.500 +I0912 11:16:24.205585 139942256621312 logging_writer.py:48] [41] global_step=41, grad_norm=0.499999, loss=6.91508 +I0912 11:16:24.208703 140000775775424 submission.py:307] 41) loss = 6.915, grad_norm = 0.500 +I0912 11:16:24.511403 139942265014016 logging_writer.py:48] [42] global_step=42, grad_norm=0.499999, loss=6.91497 +I0912 11:16:24.516593 140000775775424 submission.py:307] 42) loss = 6.915, grad_norm = 0.500 +I0912 11:16:24.803310 139942256621312 logging_writer.py:48] [43] global_step=43, grad_norm=0.499999, loss=6.91586 +I0912 11:16:24.807439 140000775775424 submission.py:307] 43) loss = 6.916, grad_norm = 0.500 +I0912 11:16:25.095148 139942265014016 logging_writer.py:48] [44] global_step=44, grad_norm=0.499999, loss=6.91515 +I0912 11:16:25.098777 140000775775424 submission.py:307] 44) loss = 6.915, grad_norm = 0.500 +I0912 11:16:25.368489 139942256621312 logging_writer.py:48] [45] global_step=45, grad_norm=0.499999, loss=6.91594 +I0912 11:16:25.372681 140000775775424 submission.py:307] 45) loss = 6.916, grad_norm = 0.500 +I0912 11:16:25.646971 139942265014016 logging_writer.py:48] [46] global_step=46, grad_norm=0.499999, loss=6.91891 +I0912 11:16:25.651124 140000775775424 submission.py:307] 46) loss = 6.919, grad_norm = 0.500 +I0912 11:16:25.925606 139942256621312 logging_writer.py:48] [47] global_step=47, grad_norm=0.499999, loss=6.90707 +I0912 11:16:25.929436 140000775775424 submission.py:307] 47) loss = 6.907, grad_norm = 0.500 +I0912 11:16:26.200562 139942265014016 logging_writer.py:48] [48] global_step=48, grad_norm=0.499999, loss=6.91347 +I0912 11:16:26.204526 140000775775424 submission.py:307] 48) loss = 6.913, grad_norm = 0.500 +I0912 11:16:26.477167 139942256621312 logging_writer.py:48] [49] global_step=49, grad_norm=0.499999, loss=6.91043 +I0912 11:16:26.481086 140000775775424 submission.py:307] 49) loss = 6.910, grad_norm = 0.500 +I0912 11:16:26.748716 139942265014016 logging_writer.py:48] [50] global_step=50, grad_norm=0.499999, loss=6.92285 +I0912 11:16:26.751804 140000775775424 submission.py:307] 50) loss = 6.923, grad_norm = 0.500 +I0912 11:16:27.040188 139942256621312 logging_writer.py:48] [51] global_step=51, grad_norm=0.499999, loss=6.91683 +I0912 11:16:27.046046 140000775775424 submission.py:307] 51) loss = 6.917, grad_norm = 0.500 +I0912 11:16:27.330578 139942265014016 logging_writer.py:48] [52] global_step=52, grad_norm=0.499999, loss=6.92897 +I0912 11:16:27.334135 140000775775424 submission.py:307] 52) loss = 6.929, grad_norm = 0.500 +I0912 11:16:27.607597 139942256621312 logging_writer.py:48] [53] global_step=53, grad_norm=0.499999, loss=6.91682 +I0912 11:16:27.611332 140000775775424 submission.py:307] 53) loss = 6.917, grad_norm = 0.500 +I0912 11:16:27.899820 139942265014016 logging_writer.py:48] [54] global_step=54, grad_norm=0.499999, loss=6.91465 +I0912 11:16:27.905354 140000775775424 submission.py:307] 54) loss = 6.915, grad_norm = 0.500 +I0912 11:16:28.181487 139942256621312 logging_writer.py:48] [55] global_step=55, grad_norm=0.499999, loss=6.91 +I0912 11:16:28.185490 140000775775424 submission.py:307] 55) loss = 6.910, grad_norm = 0.500 +I0912 11:16:28.470348 139942265014016 logging_writer.py:48] [56] global_step=56, grad_norm=0.499999, loss=6.91071 +I0912 11:16:28.475771 140000775775424 submission.py:307] 56) loss = 6.911, grad_norm = 0.500 +I0912 11:16:28.756020 139942256621312 logging_writer.py:48] [57] global_step=57, grad_norm=0.499999, loss=6.9202 +I0912 11:16:28.759985 140000775775424 submission.py:307] 57) loss = 6.920, grad_norm = 0.500 +I0912 11:16:29.031830 139942265014016 logging_writer.py:48] [58] global_step=58, grad_norm=0.499999, loss=6.91423 +I0912 11:16:29.036106 140000775775424 submission.py:307] 58) loss = 6.914, grad_norm = 0.500 +I0912 11:16:29.312865 139942256621312 logging_writer.py:48] [59] global_step=59, grad_norm=0.499999, loss=6.92204 +I0912 11:16:29.316504 140000775775424 submission.py:307] 59) loss = 6.922, grad_norm = 0.500 +I0912 11:16:29.593719 139942265014016 logging_writer.py:48] [60] global_step=60, grad_norm=0.499999, loss=6.90433 +I0912 11:16:29.598121 140000775775424 submission.py:307] 60) loss = 6.904, grad_norm = 0.500 +I0912 11:16:29.889207 139942256621312 logging_writer.py:48] [61] global_step=61, grad_norm=0.499999, loss=6.91722 +I0912 11:16:29.893238 140000775775424 submission.py:307] 61) loss = 6.917, grad_norm = 0.500 +I0912 11:16:30.165102 139942265014016 logging_writer.py:48] [62] global_step=62, grad_norm=0.499999, loss=6.92559 +I0912 11:16:30.168304 140000775775424 submission.py:307] 62) loss = 6.926, grad_norm = 0.500 +I0912 11:16:30.453688 139942256621312 logging_writer.py:48] [63] global_step=63, grad_norm=0.499999, loss=6.91311 +I0912 11:16:30.457470 140000775775424 submission.py:307] 63) loss = 6.913, grad_norm = 0.500 +I0912 11:16:30.748289 139942265014016 logging_writer.py:48] [64] global_step=64, grad_norm=0.499999, loss=6.91004 +I0912 11:16:30.751413 140000775775424 submission.py:307] 64) loss = 6.910, grad_norm = 0.500 +I0912 11:16:31.052122 139942256621312 logging_writer.py:48] [65] global_step=65, grad_norm=0.499999, loss=6.91243 +I0912 11:16:31.055550 140000775775424 submission.py:307] 65) loss = 6.912, grad_norm = 0.500 +I0912 11:16:31.332252 139942265014016 logging_writer.py:48] [66] global_step=66, grad_norm=0.499999, loss=6.90598 +I0912 11:16:31.335224 140000775775424 submission.py:307] 66) loss = 6.906, grad_norm = 0.500 +I0912 11:16:31.600108 139942256621312 logging_writer.py:48] [67] global_step=67, grad_norm=0.499999, loss=6.91894 +I0912 11:16:31.603868 140000775775424 submission.py:307] 67) loss = 6.919, grad_norm = 0.500 +I0912 11:16:31.909038 139942265014016 logging_writer.py:48] [68] global_step=68, grad_norm=0.499999, loss=6.91329 +I0912 11:16:31.915854 140000775775424 submission.py:307] 68) loss = 6.913, grad_norm = 0.500 +I0912 11:16:32.195249 139942256621312 logging_writer.py:48] [69] global_step=69, grad_norm=0.499999, loss=6.90046 +I0912 11:16:32.198371 140000775775424 submission.py:307] 69) loss = 6.900, grad_norm = 0.500 +I0912 11:16:32.492755 139942265014016 logging_writer.py:48] [70] global_step=70, grad_norm=0.499999, loss=6.90539 +I0912 11:16:32.499216 140000775775424 submission.py:307] 70) loss = 6.905, grad_norm = 0.500 +I0912 11:16:32.811412 139942256621312 logging_writer.py:48] [71] global_step=71, grad_norm=0.499999, loss=6.90368 +I0912 11:16:32.816704 140000775775424 submission.py:307] 71) loss = 6.904, grad_norm = 0.500 +I0912 11:16:33.103363 139942265014016 logging_writer.py:48] [72] global_step=72, grad_norm=0.499999, loss=6.91204 +I0912 11:16:33.106916 140000775775424 submission.py:307] 72) loss = 6.912, grad_norm = 0.500 +I0912 11:16:33.372416 139942256621312 logging_writer.py:48] [73] global_step=73, grad_norm=0.499999, loss=6.89907 +I0912 11:16:33.375894 140000775775424 submission.py:307] 73) loss = 6.899, grad_norm = 0.500 +I0912 11:16:33.662171 139942265014016 logging_writer.py:48] [74] global_step=74, grad_norm=0.499999, loss=6.90736 +I0912 11:16:33.666282 140000775775424 submission.py:307] 74) loss = 6.907, grad_norm = 0.500 +I0912 11:16:33.933192 139942256621312 logging_writer.py:48] [75] global_step=75, grad_norm=0.499999, loss=6.90346 +I0912 11:16:33.936245 140000775775424 submission.py:307] 75) loss = 6.903, grad_norm = 0.500 +I0912 11:16:34.212422 139942265014016 logging_writer.py:48] [76] global_step=76, grad_norm=0.499999, loss=6.90935 +I0912 11:16:34.216536 140000775775424 submission.py:307] 76) loss = 6.909, grad_norm = 0.500 +I0912 11:16:34.483164 139942256621312 logging_writer.py:48] [77] global_step=77, grad_norm=0.499999, loss=6.90791 +I0912 11:16:34.487215 140000775775424 submission.py:307] 77) loss = 6.908, grad_norm = 0.500 +I0912 11:16:34.785627 139942265014016 logging_writer.py:48] [78] global_step=78, grad_norm=0.499999, loss=6.90445 +I0912 11:16:34.788721 140000775775424 submission.py:307] 78) loss = 6.904, grad_norm = 0.500 +I0912 11:16:35.070803 139942256621312 logging_writer.py:48] [79] global_step=79, grad_norm=0.499999, loss=6.9104 +I0912 11:16:35.073938 140000775775424 submission.py:307] 79) loss = 6.910, grad_norm = 0.500 +I0912 11:16:35.347639 139942265014016 logging_writer.py:48] [80] global_step=80, grad_norm=0.499999, loss=6.90399 +I0912 11:16:35.351634 140000775775424 submission.py:307] 80) loss = 6.904, grad_norm = 0.500 +I0912 11:16:35.620605 139942256621312 logging_writer.py:48] [81] global_step=81, grad_norm=0.499999, loss=6.90272 +I0912 11:16:35.624792 140000775775424 submission.py:307] 81) loss = 6.903, grad_norm = 0.500 +I0912 11:16:35.916166 139942265014016 logging_writer.py:48] [82] global_step=82, grad_norm=0.499999, loss=6.8948 +I0912 11:16:35.921264 140000775775424 submission.py:307] 82) loss = 6.895, grad_norm = 0.500 +I0912 11:16:36.197182 139942256621312 logging_writer.py:48] [83] global_step=83, grad_norm=0.499999, loss=6.90069 +I0912 11:16:36.201523 140000775775424 submission.py:307] 83) loss = 6.901, grad_norm = 0.500 +I0912 11:16:36.496111 139942265014016 logging_writer.py:48] [84] global_step=84, grad_norm=0.499999, loss=6.91487 +I0912 11:16:36.499847 140000775775424 submission.py:307] 84) loss = 6.915, grad_norm = 0.500 +I0912 11:16:36.788243 139942256621312 logging_writer.py:48] [85] global_step=85, grad_norm=0.499999, loss=6.90298 +I0912 11:16:36.802633 140000775775424 submission.py:307] 85) loss = 6.903, grad_norm = 0.500 +I0912 11:16:37.100510 139942265014016 logging_writer.py:48] [86] global_step=86, grad_norm=0.499999, loss=6.90312 +I0912 11:16:37.104278 140000775775424 submission.py:307] 86) loss = 6.903, grad_norm = 0.500 +I0912 11:16:37.442716 139942256621312 logging_writer.py:48] [87] global_step=87, grad_norm=0.499999, loss=6.89799 +I0912 11:16:37.445843 140000775775424 submission.py:307] 87) loss = 6.898, grad_norm = 0.500 +I0912 11:16:37.715318 139942265014016 logging_writer.py:48] [88] global_step=88, grad_norm=0.499999, loss=6.90098 +I0912 11:16:37.719301 140000775775424 submission.py:307] 88) loss = 6.901, grad_norm = 0.500 +I0912 11:16:38.218379 139942256621312 logging_writer.py:48] [89] global_step=89, grad_norm=0.499999, loss=6.90558 +I0912 11:16:38.222650 140000775775424 submission.py:307] 89) loss = 6.906, grad_norm = 0.500 +I0912 11:16:39.020801 139942265014016 logging_writer.py:48] [90] global_step=90, grad_norm=0.499999, loss=6.91078 +I0912 11:16:39.025449 140000775775424 submission.py:307] 90) loss = 6.911, grad_norm = 0.500 +I0912 11:16:39.758610 139942256621312 logging_writer.py:48] [91] global_step=91, grad_norm=0.499999, loss=6.89075 +I0912 11:16:39.763246 140000775775424 submission.py:307] 91) loss = 6.891, grad_norm = 0.500 +I0912 11:16:40.042510 139942265014016 logging_writer.py:48] [92] global_step=92, grad_norm=0.499999, loss=6.89716 +I0912 11:16:40.045489 140000775775424 submission.py:307] 92) loss = 6.897, grad_norm = 0.500 +I0912 11:16:41.165437 139942256621312 logging_writer.py:48] [93] global_step=93, grad_norm=0.499999, loss=6.90299 +I0912 11:16:41.168352 140000775775424 submission.py:307] 93) loss = 6.903, grad_norm = 0.500 +I0912 11:16:41.486337 139942265014016 logging_writer.py:48] [94] global_step=94, grad_norm=0.499999, loss=6.90172 +I0912 11:16:41.490314 140000775775424 submission.py:307] 94) loss = 6.902, grad_norm = 0.500 +I0912 11:16:41.777461 139942256621312 logging_writer.py:48] [95] global_step=95, grad_norm=0.499999, loss=6.89168 +I0912 11:16:41.780592 140000775775424 submission.py:307] 95) loss = 6.892, grad_norm = 0.500 +I0912 11:16:42.406313 139942265014016 logging_writer.py:48] [96] global_step=96, grad_norm=0.499999, loss=6.90136 +I0912 11:16:42.414755 140000775775424 submission.py:307] 96) loss = 6.901, grad_norm = 0.500 +I0912 11:16:42.695647 139942256621312 logging_writer.py:48] [97] global_step=97, grad_norm=0.499999, loss=6.89476 +I0912 11:16:42.699487 140000775775424 submission.py:307] 97) loss = 6.895, grad_norm = 0.500 +I0912 11:16:42.971039 139942265014016 logging_writer.py:48] [98] global_step=98, grad_norm=0.499999, loss=6.8911 +I0912 11:16:42.974951 140000775775424 submission.py:307] 98) loss = 6.891, grad_norm = 0.500 +I0912 11:16:43.672982 139942256621312 logging_writer.py:48] [99] global_step=99, grad_norm=0.499999, loss=6.89461 +I0912 11:16:43.675972 140000775775424 submission.py:307] 99) loss = 6.895, grad_norm = 0.500 +I0912 11:16:43.948969 139942265014016 logging_writer.py:48] [100] global_step=100, grad_norm=0.499999, loss=6.90312 +I0912 11:16:43.952006 140000775775424 submission.py:307] 100) loss = 6.903, grad_norm = 0.500 +I0912 11:25:34.730641 139942256621312 logging_writer.py:48] [500] global_step=500, grad_norm=0.499999, loss=6.46561 +I0912 11:25:34.734242 140000775775424 submission.py:307] 500) loss = 6.466, grad_norm = 0.500 +I0912 11:36:53.972299 139942265014016 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.5, loss=5.86202 +I0912 11:36:53.976931 140000775775424 submission.py:307] 1000) loss = 5.862, grad_norm = 0.500 +I0912 11:45:09.106918 139942256621312 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.5, loss=5.43814 +I0912 11:45:09.111641 140000775775424 submission.py:307] 1500) loss = 5.438, grad_norm = 0.500 +I0912 11:49:30.842081 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 11:50:31.678468 140000775775424 spec.py:346] Evaluating on the validation split. +I0912 11:51:47.565795 140000775775424 spec.py:363] Evaluating on the test split. +I0912 11:51:49.044447 140000775775424 submission_runner.py:516] Time since start: 2534.22s, Step: 1811, {'train/accuracy': 0.07148836096938775, 'train/loss': 5.665470045440051, 'validation/accuracy': 0.07132, 'validation/loss': 5.70609875, 'validation/num_examples': 50000, 'test/accuracy': 0.0544, 'test/loss': 5.50634375, 'test/num_examples': 10000, 'score': 2180.3810358047485, 'total_duration': 2534.2220618724823, 'accumulated_submission_time': 2180.3810358047485, 'accumulated_eval_time': 349.44077491760254, 'accumulated_logging_time': 0.26671886444091797} +I0912 11:51:49.158292 139944555128576 logging_writer.py:48] [1811] accumulated_eval_time=349.441, accumulated_logging_time=0.266719, accumulated_submission_time=2180.38, global_step=1811, preemption_count=0, score=2180.38, test/accuracy=0.0544, test/loss=5.50634, test/num_examples=10000, total_duration=2534.22, train/accuracy=0.0714884, train/loss=5.66547, validation/accuracy=0.07132, validation/loss=5.7061, validation/num_examples=50000 +I0912 11:54:08.145461 139944563521280 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.5, loss=5.15686 +I0912 11:54:08.149200 140000775775424 submission.py:307] 2000) loss = 5.157, grad_norm = 0.500 +I0912 12:03:49.853128 139944555128576 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.5, loss=4.85382 +I0912 12:03:49.857685 140000775775424 submission.py:307] 2500) loss = 4.854, grad_norm = 0.500 +I0912 12:08:23.441281 139944563521280 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.5, loss=4.50813 +I0912 12:08:23.447348 140000775775424 submission.py:307] 3000) loss = 4.508, grad_norm = 0.500 +I0912 12:15:56.371632 139944555128576 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.5, loss=4.09681 +I0912 12:15:56.377136 140000775775424 submission.py:307] 3500) loss = 4.097, grad_norm = 0.500 +I0912 12:22:10.921234 139944563521280 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.5, loss=4.00937 +I0912 12:22:10.926397 140000775775424 submission.py:307] 4000) loss = 4.009, grad_norm = 0.500 +I0912 12:25:06.405956 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 12:26:04.564444 140000775775424 spec.py:346] Evaluating on the validation split. +I0912 12:27:08.893774 140000775775424 spec.py:363] Evaluating on the test split. +I0912 12:27:09.893508 140000775775424 submission_runner.py:516] Time since start: 4655.07s, Step: 4341, {'train/accuracy': 0.22741948341836735, 'train/loss': 4.783149252132493, 'validation/accuracy': 0.2126, 'validation/loss': 4.719998125, 'validation/num_examples': 50000, 'test/accuracy': 0.1697, 'test/loss': 4.324318359375, 'test/num_examples': 10000, 'score': 4173.907923221588, 'total_duration': 4655.071031808853, 'accumulated_submission_time': 4173.907923221588, 'accumulated_eval_time': 472.92827701568604, 'accumulated_logging_time': 0.38912487030029297} +I0912 12:27:10.126995 139977715283712 logging_writer.py:48] [4341] accumulated_eval_time=472.928, accumulated_logging_time=0.389125, accumulated_submission_time=4173.91, global_step=4341, preemption_count=0, score=4173.91, test/accuracy=0.1697, test/loss=4.32432, test/num_examples=10000, total_duration=4655.07, train/accuracy=0.227419, train/loss=4.78315, validation/accuracy=0.2126, validation/loss=4.72, validation/num_examples=50000 +I0912 12:28:35.474146 139977799210752 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.5, loss=3.51828 +I0912 12:28:35.478185 140000775775424 submission.py:307] 4500) loss = 3.518, grad_norm = 0.500 +I0912 12:36:56.176650 139977715283712 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.5, loss=3.45812 +I0912 12:36:56.181142 140000775775424 submission.py:307] 5000) loss = 3.458, grad_norm = 0.500 +I0912 12:40:46.564275 139977799210752 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.5, loss=3.32824 +I0912 12:40:46.568633 140000775775424 submission.py:307] 5500) loss = 3.328, grad_norm = 0.500 +I0912 12:47:00.664054 139977715283712 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.5, loss=3.00161 +I0912 12:47:00.671276 140000775775424 submission.py:307] 6000) loss = 3.002, grad_norm = 0.500 +I0912 12:52:46.949220 139977799210752 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.5, loss=2.86401 +I0912 12:52:46.960209 140000775775424 submission.py:307] 6500) loss = 2.864, grad_norm = 0.500 +I0912 12:57:12.569653 139977715283712 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.5, loss=2.88004 +I0912 12:57:12.574150 140000775775424 submission.py:307] 7000) loss = 2.880, grad_norm = 0.500 +I0912 13:00:29.667662 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 13:01:10.570861 140000775775424 spec.py:346] Evaluating on the validation split. +I0912 13:02:31.662859 140000775775424 spec.py:363] Evaluating on the test split. +I0912 13:02:32.813108 140000775775424 submission_runner.py:516] Time since start: 6777.99s, Step: 7220, {'train/accuracy': 0.3622249681122449, 'train/loss': 4.767920046436544, 'validation/accuracy': 0.36892, 'validation/loss': 3.9849725, 'validation/num_examples': 50000, 'test/accuracy': 0.3084, 'test/loss': 3.45687734375, 'test/num_examples': 10000, 'score': 6168.158279180527, 'total_duration': 6777.990676403046, 'accumulated_submission_time': 6168.158279180527, 'accumulated_eval_time': 596.0737833976746, 'accumulated_logging_time': 0.6317288875579834} +I0912 13:02:33.063308 139977740461824 logging_writer.py:48] [7220] accumulated_eval_time=596.074, accumulated_logging_time=0.631729, accumulated_submission_time=6168.16, global_step=7220, preemption_count=0, score=6168.16, test/accuracy=0.3084, test/loss=3.45688, test/num_examples=10000, total_duration=6777.99, train/accuracy=0.362225, train/loss=4.76792, validation/accuracy=0.36892, validation/loss=3.98497, validation/num_examples=50000 +I0912 13:06:09.640642 139977807603456 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.5, loss=2.69043 +I0912 13:06:09.644170 140000775775424 submission.py:307] 7500) loss = 2.690, grad_norm = 0.500 +I0912 13:10:09.756854 139977740461824 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.5, loss=2.58819 +I0912 13:10:09.761046 140000775775424 submission.py:307] 8000) loss = 2.588, grad_norm = 0.500 +I0912 13:16:41.855081 139977807603456 logging_writer.py:48] [8500] global_step=8500, grad_norm=0.5, loss=2.43703 +I0912 13:16:41.861273 140000775775424 submission.py:307] 8500) loss = 2.437, grad_norm = 0.500 +I0912 13:22:21.202527 139977740461824 logging_writer.py:48] [9000] global_step=9000, grad_norm=0.5, loss=2.45517 +I0912 13:22:21.209443 140000775775424 submission.py:307] 9000) loss = 2.455, grad_norm = 0.500 +I0912 13:26:48.680655 139977807603456 logging_writer.py:48] [9500] global_step=9500, grad_norm=0.5, loss=2.29047 +I0912 13:26:48.687087 140000775775424 submission.py:307] 9500) loss = 2.290, grad_norm = 0.500 +I0912 13:34:34.755501 139977740461824 logging_writer.py:48] [10000] global_step=10000, grad_norm=0.5, loss=2.15092 +I0912 13:34:34.760844 140000775775424 submission.py:307] 10000) loss = 2.151, grad_norm = 0.500 +I0912 13:35:50.267681 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 13:36:30.124269 140000775775424 spec.py:346] Evaluating on the validation split. +I0912 13:37:36.985527 140000775775424 spec.py:363] Evaluating on the test split. +I0912 13:37:37.881951 140000775775424 submission_runner.py:516] Time since start: 8883.06s, Step: 10186, {'train/accuracy': 0.4917091836734694, 'train/loss': 3.8733293182995854, 'validation/accuracy': 0.46566, 'validation/loss': 3.754048125, 'validation/num_examples': 50000, 'test/accuracy': 0.3987, 'test/loss': 2.8301828125, 'test/num_examples': 10000, 'score': 8158.115170717239, 'total_duration': 8883.059608697891, 'accumulated_submission_time': 8158.115170717239, 'accumulated_eval_time': 703.6881942749023, 'accumulated_logging_time': 0.8909380435943604} +I0912 13:37:38.097396 139977757247232 logging_writer.py:48] [10186] accumulated_eval_time=703.688, accumulated_logging_time=0.890938, accumulated_submission_time=8158.12, global_step=10186, preemption_count=0, score=8158.12, test/accuracy=0.3987, test/loss=2.83018, test/num_examples=10000, total_duration=8883.06, train/accuracy=0.491709, train/loss=3.87333, validation/accuracy=0.46566, validation/loss=3.75405, validation/num_examples=50000 +I0912 13:40:15.818460 139977782425344 logging_writer.py:48] [10500] global_step=10500, grad_norm=0.5, loss=2.26976 +I0912 13:40:15.823584 140000775775424 submission.py:307] 10500) loss = 2.270, grad_norm = 0.500 +I0912 13:47:05.082211 139977757247232 logging_writer.py:48] [11000] global_step=11000, grad_norm=0.5, loss=2.10826 +I0912 13:47:05.085886 140000775775424 submission.py:307] 11000) loss = 2.108, grad_norm = 0.500 +I0912 13:53:02.253979 139977782425344 logging_writer.py:48] [11500] global_step=11500, grad_norm=0.5, loss=1.98107 +I0912 13:53:02.290371 140000775775424 submission.py:307] 11500) loss = 1.981, grad_norm = 0.500 +I0912 13:57:31.250774 139977757247232 logging_writer.py:48] [12000] global_step=12000, grad_norm=0.5, loss=1.98456 +I0912 13:57:31.350964 140000775775424 submission.py:307] 12000) loss = 1.985, grad_norm = 0.500 +I0912 14:05:24.893883 139977782425344 logging_writer.py:48] [12500] global_step=12500, grad_norm=0.5, loss=2.10933 +I0912 14:05:25.169235 140000775775424 submission.py:307] 12500) loss = 2.109, grad_norm = 0.500 +I0912 14:08:57.062365 139977757247232 logging_writer.py:48] [13000] global_step=13000, grad_norm=0.5, loss=1.94113 +I0912 14:08:57.066840 140000775775424 submission.py:307] 13000) loss = 1.941, grad_norm = 0.500 +I0912 14:10:58.820405 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 14:11:49.656169 140000775775424 spec.py:346] Evaluating on the validation split. +I0912 14:12:57.041985 140000775775424 spec.py:363] Evaluating on the test split. +I0912 14:12:57.978430 140000775775424 submission_runner.py:516] Time since start: 11003.16s, Step: 13198, {'train/accuracy': 0.5372090242346939, 'train/loss': 4.049487055564414, 'validation/accuracy': 0.50746, 'validation/loss': 3.7652409375, 'validation/num_examples': 50000, 'test/accuracy': 0.4514, 'test/loss': 2.54749296875, 'test/num_examples': 10000, 'score': 10148.827378749847, 'total_duration': 11003.156066417694, 'accumulated_submission_time': 10148.827378749847, 'accumulated_eval_time': 822.8465011119843, 'accumulated_logging_time': 1.123784065246582} +I0912 14:12:58.274718 139977815996160 logging_writer.py:48] [13198] accumulated_eval_time=822.847, accumulated_logging_time=1.12378, accumulated_submission_time=10148.8, global_step=13198, preemption_count=0, score=10148.8, test/accuracy=0.4514, test/loss=2.54749, test/num_examples=10000, total_duration=11003.2, train/accuracy=0.537209, train/loss=4.04949, validation/accuracy=0.50746, validation/loss=3.76524, validation/num_examples=50000 +I0912 14:16:50.859838 139977723676416 logging_writer.py:48] [13500] global_step=13500, grad_norm=0.5, loss=1.85869 +I0912 14:16:50.863660 140000775775424 submission.py:307] 13500) loss = 1.859, grad_norm = 0.500 +I0912 14:22:48.834769 139977815996160 logging_writer.py:48] [14000] global_step=14000, grad_norm=0.5, loss=1.85497 +I0912 14:22:48.839486 140000775775424 submission.py:307] 14000) loss = 1.855, grad_norm = 0.500 +I0912 14:27:37.860855 139977723676416 logging_writer.py:48] [14500] global_step=14500, grad_norm=0.5, loss=1.91415 +I0912 14:27:38.582486 140000775775424 submission.py:307] 14500) loss = 1.914, grad_norm = 0.500 +I0912 14:35:21.219466 139977815996160 logging_writer.py:48] [15000] global_step=15000, grad_norm=0.5, loss=1.80982 +I0912 14:35:21.224141 140000775775424 submission.py:307] 15000) loss = 1.810, grad_norm = 0.500 +I0912 14:38:50.342396 139977723676416 logging_writer.py:48] [15500] global_step=15500, grad_norm=0.5, loss=1.72106 +I0912 14:38:50.347497 140000775775424 submission.py:307] 15500) loss = 1.721, grad_norm = 0.500 +I0912 14:45:16.210345 139977815996160 logging_writer.py:48] [16000] global_step=16000, grad_norm=0.5, loss=1.76659 +I0912 14:45:16.214293 140000775775424 submission.py:307] 16000) loss = 1.767, grad_norm = 0.500 +I0912 14:46:20.472733 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 14:46:59.044920 140000775775424 spec.py:346] Evaluating on the validation split. +I0912 14:48:13.977795 140000775775424 spec.py:363] Evaluating on the test split. +I0912 14:48:15.027339 140000775775424 submission_runner.py:516] Time since start: 13120.20s, Step: 16058, {'train/accuracy': 0.5820910395408163, 'train/loss': 3.6331323117625955, 'validation/accuracy': 0.53814, 'validation/loss': 3.2271865625, 'validation/num_examples': 50000, 'test/accuracy': 0.4753, 'test/loss': 2.3972625, 'test/num_examples': 10000, 'score': 12142.09969496727, 'total_duration': 13120.20466184616, 'accumulated_submission_time': 12142.09969496727, 'accumulated_eval_time': 937.401035785675, 'accumulated_logging_time': 1.4320638179779053} +I0912 14:48:15.340585 139977757247232 logging_writer.py:48] [16058] accumulated_eval_time=937.401, accumulated_logging_time=1.43206, accumulated_submission_time=12142.1, global_step=16058, preemption_count=0, score=12142.1, test/accuracy=0.4753, test/loss=2.39726, test/num_examples=10000, total_duration=13120.2, train/accuracy=0.582091, train/loss=3.63313, validation/accuracy=0.53814, validation/loss=3.22719, validation/num_examples=50000 +I0912 14:52:24.174791 139977740461824 logging_writer.py:48] [16500] global_step=16500, grad_norm=0.5, loss=1.79579 +I0912 14:52:24.179014 140000775775424 submission.py:307] 16500) loss = 1.796, grad_norm = 0.500 +I0912 14:57:31.722265 139977757247232 logging_writer.py:48] [17000] global_step=17000, grad_norm=0.5, loss=1.72101 +I0912 14:57:31.730538 140000775775424 submission.py:307] 17000) loss = 1.721, grad_norm = 0.500 +I0912 15:05:25.588392 139977740461824 logging_writer.py:48] [17500] global_step=17500, grad_norm=0.5, loss=1.70501 +I0912 15:05:25.594466 140000775775424 submission.py:307] 17500) loss = 1.705, grad_norm = 0.500 +I0912 15:08:58.894270 139977757247232 logging_writer.py:48] [18000] global_step=18000, grad_norm=0.5, loss=1.61612 +I0912 15:08:58.898634 140000775775424 submission.py:307] 18000) loss = 1.616, grad_norm = 0.500 +I0912 15:15:29.202468 139977740461824 logging_writer.py:48] [18500] global_step=18500, grad_norm=0.5, loss=1.73051 +I0912 15:15:29.207477 140000775775424 submission.py:307] 18500) loss = 1.731, grad_norm = 0.500 +I0912 15:21:20.172914 139977757247232 logging_writer.py:48] [19000] global_step=19000, grad_norm=0.5, loss=1.60282 +I0912 15:21:20.184099 140000775775424 submission.py:307] 19000) loss = 1.603, grad_norm = 0.500 +I0912 15:21:33.611403 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 15:22:17.191391 140000775775424 spec.py:346] Evaluating on the validation split. +I0912 15:23:24.351184 140000775775424 spec.py:363] Evaluating on the test split. +I0912 15:23:25.304350 140000775775424 submission_runner.py:516] Time since start: 15230.48s, Step: 19027, {'train/accuracy': 0.6128427933673469, 'train/loss': 3.5851770517777424, 'validation/accuracy': 0.55534, 'validation/loss': 3.4209021875, 'validation/num_examples': 50000, 'test/accuracy': 0.4994, 'test/loss': 2.3539578125, 'test/num_examples': 10000, 'score': 14133.677445173264, 'total_duration': 15230.481875181198, 'accumulated_submission_time': 14133.677445173264, 'accumulated_eval_time': 1049.0939209461212, 'accumulated_logging_time': 1.754276990890503} +I0912 15:23:25.644815 139977841174272 logging_writer.py:48] [19027] accumulated_eval_time=1049.09, accumulated_logging_time=1.75428, accumulated_submission_time=14133.7, global_step=19027, preemption_count=0, score=14133.7, test/accuracy=0.4994, test/loss=2.35396, test/num_examples=10000, total_duration=15230.5, train/accuracy=0.612843, train/loss=3.58518, validation/accuracy=0.55534, validation/loss=3.4209, validation/num_examples=50000 +I0912 15:28:15.335289 139977790818048 logging_writer.py:48] [19500] global_step=19500, grad_norm=0.5, loss=1.53031 +I0912 15:28:15.339933 140000775775424 submission.py:307] 19500) loss = 1.530, grad_norm = 0.500 +I0912 15:36:33.865808 139977841174272 logging_writer.py:48] [20000] global_step=20000, grad_norm=0.5, loss=1.59319 +I0912 15:36:33.870478 140000775775424 submission.py:307] 20000) loss = 1.593, grad_norm = 0.500 +I0912 15:40:06.108171 139977790818048 logging_writer.py:48] [20500] global_step=20500, grad_norm=0.5, loss=1.59517 +I0912 15:40:06.125305 140000775775424 submission.py:307] 20500) loss = 1.595, grad_norm = 0.500 +I0912 15:46:36.377521 139977841174272 logging_writer.py:48] [21000] global_step=21000, grad_norm=0.5, loss=1.52382 +I0912 15:46:36.381595 140000775775424 submission.py:307] 21000) loss = 1.524, grad_norm = 0.500 +I0912 15:52:26.839221 139977790818048 logging_writer.py:48] [21500] global_step=21500, grad_norm=0.5, loss=1.53113 +I0912 15:52:26.857593 140000775775424 submission.py:307] 21500) loss = 1.531, grad_norm = 0.500 +I0912 15:56:43.006755 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 15:57:36.036365 140000775775424 spec.py:346] Evaluating on the validation split. +I0912 15:58:46.088955 140000775775424 spec.py:363] Evaluating on the test split. +I0912 15:58:47.337822 140000775775424 submission_runner.py:516] Time since start: 17352.52s, Step: 21984, {'train/accuracy': 0.6188815369897959, 'train/loss': 3.9348521329918684, 'validation/accuracy': 0.5705, 'validation/loss': 3.4900921875, 'validation/num_examples': 50000, 'test/accuracy': 0.515, 'test/loss': 2.26014921875, 'test/num_examples': 10000, 'score': 16124.675315141678, 'total_duration': 17352.515357732773, 'accumulated_submission_time': 16124.675315141678, 'accumulated_eval_time': 1173.4251527786255, 'accumulated_logging_time': 2.109804153442383} +I0912 15:58:47.695386 139977757247232 logging_writer.py:48] [21984] accumulated_eval_time=1173.43, accumulated_logging_time=2.1098, accumulated_submission_time=16124.7, global_step=21984, preemption_count=0, score=16124.7, test/accuracy=0.515, test/loss=2.26015, test/num_examples=10000, total_duration=17352.5, train/accuracy=0.618882, train/loss=3.93485, validation/accuracy=0.5705, validation/loss=3.49009, validation/num_examples=50000 +I0912 15:58:55.061501 139977832781568 logging_writer.py:48] [22000] global_step=22000, grad_norm=0.5, loss=1.47937 +I0912 15:58:55.064711 140000775775424 submission.py:307] 22000) loss = 1.479, grad_norm = 0.500 +I0912 16:06:15.315718 139977757247232 logging_writer.py:48] [22500] global_step=22500, grad_norm=0.5, loss=1.5886 +I0912 16:06:15.319531 140000775775424 submission.py:307] 22500) loss = 1.589, grad_norm = 0.500 +I0912 16:10:21.304990 139977832781568 logging_writer.py:48] [23000] global_step=23000, grad_norm=0.5, loss=1.53766 +I0912 16:10:21.310106 140000775775424 submission.py:307] 23000) loss = 1.538, grad_norm = 0.500 +I0912 16:16:41.367340 139977757247232 logging_writer.py:48] [23500] global_step=23500, grad_norm=0.5, loss=1.42024 +I0912 16:16:41.373373 140000775775424 submission.py:307] 23500) loss = 1.420, grad_norm = 0.500 +I0912 16:22:30.220772 139977832781568 logging_writer.py:48] [24000] global_step=24000, grad_norm=0.5, loss=1.3955 +I0912 16:22:30.230311 140000775775424 submission.py:307] 24000) loss = 1.395, grad_norm = 0.500 +I0912 16:27:02.606581 139977757247232 logging_writer.py:48] [24500] global_step=24500, grad_norm=0.5, loss=1.43699 +I0912 16:27:02.611894 140000775775424 submission.py:307] 24500) loss = 1.437, grad_norm = 0.500 +I0912 16:32:08.598855 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 16:32:47.715197 140000775775424 spec.py:346] Evaluating on the validation split. +I0912 16:34:08.340837 140000775775424 spec.py:363] Evaluating on the test split. +I0912 16:34:09.310796 140000775775424 submission_runner.py:516] Time since start: 19474.49s, Step: 24826, {'train/accuracy': 0.6424784757653061, 'train/loss': 3.722700780751754, 'validation/accuracy': 0.58654, 'validation/loss': 3.1755021875, 'validation/num_examples': 50000, 'test/accuracy': 0.5172, 'test/loss': 2.265676171875, 'test/num_examples': 10000, 'score': 18115.26232433319, 'total_duration': 19474.487339258194, 'accumulated_submission_time': 18115.26232433319, 'accumulated_eval_time': 1294.1361372470856, 'accumulated_logging_time': 2.48134183883667} +I0912 16:34:09.722120 139977732069120 logging_writer.py:48] [24826] accumulated_eval_time=1294.14, accumulated_logging_time=2.48134, accumulated_submission_time=18115.3, global_step=24826, preemption_count=0, score=18115.3, test/accuracy=0.5172, test/loss=2.26568, test/num_examples=10000, total_duration=19474.5, train/accuracy=0.642478, train/loss=3.7227, validation/accuracy=0.58654, validation/loss=3.1755, validation/num_examples=50000 +I0912 16:36:06.614065 139977832781568 logging_writer.py:48] [25000] global_step=25000, grad_norm=0.5, loss=1.46616 +I0912 16:36:06.617811 140000775775424 submission.py:307] 25000) loss = 1.466, grad_norm = 0.500 +I0912 16:40:17.375005 139977732069120 logging_writer.py:48] [25500] global_step=25500, grad_norm=0.5, loss=1.44165 +I0912 16:40:17.382875 140000775775424 submission.py:307] 25500) loss = 1.442, grad_norm = 0.500 +I0912 16:47:03.384861 139977832781568 logging_writer.py:48] [26000] global_step=26000, grad_norm=0.5, loss=1.39321 +I0912 16:47:03.390290 140000775775424 submission.py:307] 26000) loss = 1.393, grad_norm = 0.500 +I0912 16:52:51.506302 139977732069120 logging_writer.py:48] [26500] global_step=26500, grad_norm=0.5, loss=1.40953 +I0912 16:52:51.512306 140000775775424 submission.py:307] 26500) loss = 1.410, grad_norm = 0.500 +I0912 16:57:17.871438 139977832781568 logging_writer.py:48] [27000] global_step=27000, grad_norm=0.5, loss=1.4462 +I0912 16:57:17.875375 140000775775424 submission.py:307] 27000) loss = 1.446, grad_norm = 0.500 +I0912 17:05:24.409037 139977732069120 logging_writer.py:48] [27500] global_step=27500, grad_norm=0.5, loss=1.2602 +I0912 17:05:24.415585 140000775775424 submission.py:307] 27500) loss = 1.260, grad_norm = 0.500 +I0912 17:07:26.973580 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 17:08:13.285521 140000775775424 spec.py:346] Evaluating on the validation split. +I0912 17:09:20.006819 140000775775424 spec.py:363] Evaluating on the test split. +I0912 17:09:20.998187 140000775775424 submission_runner.py:516] Time since start: 21586.18s, Step: 27803, {'train/accuracy': 0.6767976721938775, 'train/loss': 3.3712301448899873, 'validation/accuracy': 0.582, 'validation/loss': 3.53975875, 'validation/num_examples': 50000, 'test/accuracy': 0.5188, 'test/loss': 2.329924609375, 'test/num_examples': 10000, 'score': 20106.77537536621, 'total_duration': 21586.175618886948, 'accumulated_submission_time': 20106.77537536621, 'accumulated_eval_time': 1408.1605834960938, 'accumulated_logging_time': 2.901373863220215} +I0912 17:09:21.392075 139977698498304 logging_writer.py:48] [27803] accumulated_eval_time=1408.16, accumulated_logging_time=2.90137, accumulated_submission_time=20106.8, global_step=27803, preemption_count=0, score=20106.8, test/accuracy=0.5188, test/loss=2.32992, test/num_examples=10000, total_duration=21586.2, train/accuracy=0.676798, train/loss=3.37123, validation/accuracy=0.582, validation/loss=3.53976, validation/num_examples=50000 +I0912 17:10:53.588807 139977832781568 logging_writer.py:48] [28000] global_step=28000, grad_norm=0.5, loss=1.42421 +I0912 17:10:53.596899 140000775775424 submission.py:307] 28000) loss = 1.424, grad_norm = 0.500 +I0912 17:17:55.825262 139977698498304 logging_writer.py:48] [28500] global_step=28500, grad_norm=0.5, loss=1.42829 +I0912 17:17:55.829233 140000775775424 submission.py:307] 28500) loss = 1.428, grad_norm = 0.500 +I0912 17:24:01.392967 139977832781568 logging_writer.py:48] [29000] global_step=29000, grad_norm=0.5, loss=1.28602 +I0912 17:24:01.397018 140000775775424 submission.py:307] 29000) loss = 1.286, grad_norm = 0.500 +I0912 17:28:34.232821 139977698498304 logging_writer.py:48] [29500] global_step=29500, grad_norm=0.5, loss=1.28162 +I0912 17:28:34.237620 140000775775424 submission.py:307] 29500) loss = 1.282, grad_norm = 0.500 +I0912 17:36:38.697960 139977832781568 logging_writer.py:48] [30000] global_step=30000, grad_norm=0.5, loss=1.28172 +I0912 17:36:38.701691 140000775775424 submission.py:307] 30000) loss = 1.282, grad_norm = 0.500 +I0912 17:40:08.177314 139977698498304 logging_writer.py:48] [30500] global_step=30500, grad_norm=0.5, loss=1.28083 +I0912 17:40:08.181256 140000775775424 submission.py:307] 30500) loss = 1.281, grad_norm = 0.500 +I0912 17:42:41.416619 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 17:43:34.126795 140000775775424 spec.py:346] Evaluating on the validation split. +I0912 17:44:42.532206 140000775775424 spec.py:363] Evaluating on the test split. +I0912 17:44:43.762040 140000775775424 submission_runner.py:516] Time since start: 23708.94s, Step: 30744, {'train/accuracy': 0.682258450255102, 'train/loss': 3.395152812101403, 'validation/accuracy': 0.597, 'validation/loss': 3.24307375, 'validation/num_examples': 50000, 'test/accuracy': 0.5315, 'test/loss': 2.2064314453125, 'test/num_examples': 10000, 'score': 22101.098651885986, 'total_duration': 23708.939203500748, 'accumulated_submission_time': 22101.098651885986, 'accumulated_eval_time': 1530.5059394836426, 'accumulated_logging_time': 3.3105220794677734} +I0912 17:44:44.098983 139977706891008 logging_writer.py:48] [30744] accumulated_eval_time=1530.51, accumulated_logging_time=3.31052, accumulated_submission_time=22101.1, global_step=30744, preemption_count=0, score=22101.1, test/accuracy=0.5315, test/loss=2.20643, test/num_examples=10000, total_duration=23708.9, train/accuracy=0.682258, train/loss=3.39515, validation/accuracy=0.597, validation/loss=3.24307, validation/num_examples=50000 +I0912 17:47:49.147865 139977807603456 logging_writer.py:48] [31000] global_step=31000, grad_norm=0.5, loss=1.2526 +I0912 17:47:49.152586 140000775775424 submission.py:307] 31000) loss = 1.253, grad_norm = 0.500 +I0912 17:54:04.390127 139977706891008 logging_writer.py:48] [31500] global_step=31500, grad_norm=0.5, loss=1.21339 +I0912 17:54:04.395957 140000775775424 submission.py:307] 31500) loss = 1.213, grad_norm = 0.500 +I0912 17:58:52.366230 139977807603456 logging_writer.py:48] [32000] global_step=32000, grad_norm=0.5, loss=1.24288 +I0912 17:58:52.370923 140000775775424 submission.py:307] 32000) loss = 1.243, grad_norm = 0.500 +I0912 18:06:46.374420 139977706891008 logging_writer.py:48] [32500] global_step=32500, grad_norm=0.5, loss=1.40237 +I0912 18:06:46.378443 140000775775424 submission.py:307] 32500) loss = 1.402, grad_norm = 0.500 +I0912 18:10:14.608919 139977807603456 logging_writer.py:48] [33000] global_step=33000, grad_norm=0.5, loss=1.29882 +I0912 18:10:14.614486 140000775775424 submission.py:307] 33000) loss = 1.299, grad_norm = 0.500 +I0912 18:16:28.503386 139977706891008 logging_writer.py:48] [33500] global_step=33500, grad_norm=0.5, loss=1.27991 +I0912 18:16:28.509570 140000775775424 submission.py:307] 33500) loss = 1.280, grad_norm = 0.500 +I0912 18:18:12.254811 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 18:18:51.352938 140000775775424 spec.py:346] Evaluating on the validation split. +I0912 18:20:15.720007 140000775775424 spec.py:363] Evaluating on the test split. +I0912 18:20:16.890708 140000775775424 submission_runner.py:516] Time since start: 25842.07s, Step: 33587, {'train/accuracy': 0.6550542091836735, 'train/loss': 4.210914456114477, 'validation/accuracy': 0.5883, 'validation/loss': 3.418475625, 'validation/num_examples': 50000, 'test/accuracy': 0.5299, 'test/loss': 2.2689298828125, 'test/num_examples': 10000, 'score': 24099.835663318634, 'total_duration': 25842.068258285522, 'accumulated_submission_time': 24099.835663318634, 'accumulated_eval_time': 1655.1419308185577, 'accumulated_logging_time': 3.6563520431518555} +I0912 18:20:17.203388 139965144938240 logging_writer.py:48] [33587] accumulated_eval_time=1655.14, accumulated_logging_time=3.65635, accumulated_submission_time=24099.8, global_step=33587, preemption_count=0, score=24099.8, test/accuracy=0.5299, test/loss=2.26893, test/num_examples=10000, total_duration=25842.1, train/accuracy=0.655054, train/loss=4.21091, validation/accuracy=0.5883, validation/loss=3.41848, validation/num_examples=50000 +I0912 18:24:05.130343 139977807603456 logging_writer.py:48] [34000] global_step=34000, grad_norm=0.5, loss=1.23143 +I0912 18:24:05.137028 140000775775424 submission.py:307] 34000) loss = 1.231, grad_norm = 0.500 +I0912 18:29:18.446489 139965144938240 logging_writer.py:48] [34500] global_step=34500, grad_norm=0.5, loss=1.22161 +I0912 18:29:18.451255 140000775775424 submission.py:307] 34500) loss = 1.222, grad_norm = 0.500 +I0912 18:37:13.455294 139977807603456 logging_writer.py:48] [35000] global_step=35000, grad_norm=0.5, loss=1.22477 +I0912 18:37:13.459369 140000775775424 submission.py:307] 35000) loss = 1.225, grad_norm = 0.500 +I0912 18:40:43.710355 139965144938240 logging_writer.py:48] [35500] global_step=35500, grad_norm=0.5, loss=1.24787 +I0912 18:40:43.716703 140000775775424 submission.py:307] 35500) loss = 1.248, grad_norm = 0.500 +I0912 18:47:08.432425 139977807603456 logging_writer.py:48] [36000] global_step=36000, grad_norm=0.5, loss=1.14906 +I0912 18:47:08.437962 140000775775424 submission.py:307] 36000) loss = 1.149, grad_norm = 0.500 +I0912 18:53:10.313524 139965144938240 logging_writer.py:48] [36500] global_step=36500, grad_norm=0.5, loss=1.24151 +I0912 18:53:10.320133 140000775775424 submission.py:307] 36500) loss = 1.242, grad_norm = 0.500 +I0912 18:53:36.266494 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 18:54:20.166743 140000775775424 spec.py:346] Evaluating on the validation split. +I0912 18:55:26.925627 140000775775424 spec.py:363] Evaluating on the test split. +I0912 18:55:27.919239 140000775775424 submission_runner.py:516] Time since start: 27953.10s, Step: 36561, {'train/accuracy': 0.6913066007653061, 'train/loss': 3.5630960269850127, 'validation/accuracy': 0.5924, 'validation/loss': 3.4600703125, 'validation/num_examples': 50000, 'test/accuracy': 0.5332, 'test/loss': 2.25507109375, 'test/num_examples': 10000, 'score': 26091.93958926201, 'total_duration': 27953.096845149994, 'accumulated_submission_time': 26091.93958926201, 'accumulated_eval_time': 1766.7947697639465, 'accumulated_logging_time': 3.978273391723633} +I0912 18:55:28.232630 139977841174272 logging_writer.py:48] [36561] accumulated_eval_time=1766.79, accumulated_logging_time=3.97827, accumulated_submission_time=26091.9, global_step=36561, preemption_count=0, score=26091.9, test/accuracy=0.5332, test/loss=2.25507, test/num_examples=10000, total_duration=27953.1, train/accuracy=0.691307, train/loss=3.5631, validation/accuracy=0.5924, validation/loss=3.46007, validation/num_examples=50000 +I0912 18:59:54.373818 139977706891008 logging_writer.py:48] [37000] global_step=37000, grad_norm=0.5, loss=1.15111 +I0912 18:59:54.378173 140000775775424 submission.py:307] 37000) loss = 1.151, grad_norm = 0.500 +I0912 19:08:19.627212 139977841174272 logging_writer.py:48] [37500] global_step=37500, grad_norm=0.5, loss=1.22178 +I0912 19:08:19.631018 140000775775424 submission.py:307] 37500) loss = 1.222, grad_norm = 0.500 +I0912 19:11:57.092137 139977706891008 logging_writer.py:48] [38000] global_step=38000, grad_norm=0.5, loss=1.16999 +I0912 19:11:57.096110 140000775775424 submission.py:307] 38000) loss = 1.170, grad_norm = 0.500 +I0912 19:18:19.582246 139977841174272 logging_writer.py:48] [38500] global_step=38500, grad_norm=0.5, loss=1.20513 +I0912 19:18:19.588173 140000775775424 submission.py:307] 38500) loss = 1.205, grad_norm = 0.500 +I0912 19:24:15.695069 139977706891008 logging_writer.py:48] [39000] global_step=39000, grad_norm=0.5, loss=1.06604 +I0912 19:24:15.738178 140000775775424 submission.py:307] 39000) loss = 1.066, grad_norm = 0.500 +I0912 19:28:35.901168 139977841174272 logging_writer.py:48] [39500] global_step=39500, grad_norm=0.5, loss=1.18123 +I0912 19:28:35.906233 140000775775424 submission.py:307] 39500) loss = 1.181, grad_norm = 0.500 +I0912 19:28:46.807513 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 19:29:38.108703 140000775775424 spec.py:346] Evaluating on the validation split. +I0912 19:30:48.067546 140000775775424 spec.py:363] Evaluating on the test split. +I0912 19:30:49.346199 140000775775424 submission_runner.py:516] Time since start: 30074.52s, Step: 39508, {'train/accuracy': 0.6902901785714286, 'train/loss': 4.021324313416773, 'validation/accuracy': 0.58788, 'validation/loss': 3.737465, 'validation/num_examples': 50000, 'test/accuracy': 0.5406, 'test/loss': 2.2091263671875, 'test/num_examples': 10000, 'score': 28084.79198908806, 'total_duration': 30074.523801088333, 'accumulated_submission_time': 28084.79198908806, 'accumulated_eval_time': 1889.3335394859314, 'accumulated_logging_time': 4.304686069488525} +I0912 19:30:49.631033 139977748854528 logging_writer.py:48] [39508] accumulated_eval_time=1889.33, accumulated_logging_time=4.30469, accumulated_submission_time=28084.8, global_step=39508, preemption_count=0, score=28084.8, test/accuracy=0.5406, test/loss=2.20913, test/num_examples=10000, total_duration=30074.5, train/accuracy=0.69029, train/loss=4.02132, validation/accuracy=0.58788, validation/loss=3.73746, validation/num_examples=50000 +I0912 19:38:01.770733 139977799210752 logging_writer.py:48] [40000] global_step=40000, grad_norm=0.5, loss=1.18884 +I0912 19:38:01.775496 140000775775424 submission.py:307] 40000) loss = 1.189, grad_norm = 0.500 +I0912 19:42:15.337757 139977748854528 logging_writer.py:48] [40500] global_step=40500, grad_norm=0.5, loss=1.21566 +I0912 19:42:15.341996 140000775775424 submission.py:307] 40500) loss = 1.216, grad_norm = 0.500 +I0912 19:48:41.120955 139977799210752 logging_writer.py:48] [41000] global_step=41000, grad_norm=0.5, loss=1.16538 +I0912 19:48:41.136950 140000775775424 submission.py:307] 41000) loss = 1.165, grad_norm = 0.500 +I0912 19:54:31.011373 139977748854528 logging_writer.py:48] [41500] global_step=41500, grad_norm=0.5, loss=1.07511 +I0912 19:54:31.018791 140000775775424 submission.py:307] 41500) loss = 1.075, grad_norm = 0.500 +I0912 19:58:51.731662 139977799210752 logging_writer.py:48] [42000] global_step=42000, grad_norm=0.5, loss=1.12688 +I0912 19:58:51.736583 140000775775424 submission.py:307] 42000) loss = 1.127, grad_norm = 0.500 +I0912 20:04:10.870386 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 20:04:48.594920 140000775775424 spec.py:346] Evaluating on the validation split. +I0912 20:06:09.908089 140000775775424 spec.py:363] Evaluating on the test split. +I0912 20:06:11.206268 140000775775424 submission_runner.py:516] Time since start: 32196.38s, Step: 42336, {'train/accuracy': 0.6846500318877551, 'train/loss': 4.242830237563775, 'validation/accuracy': 0.59668, 'validation/loss': 3.5448959375, 'validation/num_examples': 50000, 'test/accuracy': 0.5405, 'test/loss': 2.2322970703125, 'test/num_examples': 10000, 'score': 30076.157195091248, 'total_duration': 32196.383840560913, 'accumulated_submission_time': 30076.157195091248, 'accumulated_eval_time': 2009.6694073677063, 'accumulated_logging_time': 4.598609685897827} +I0912 20:06:11.543324 139977832781568 logging_writer.py:48] [42336] accumulated_eval_time=2009.67, accumulated_logging_time=4.59861, accumulated_submission_time=30076.2, global_step=42336, preemption_count=0, score=30076.2, test/accuracy=0.5405, test/loss=2.2323, test/num_examples=10000, total_duration=32196.4, train/accuracy=0.68465, train/loss=4.24283, validation/accuracy=0.59668, validation/loss=3.5449, validation/num_examples=50000 +I0912 20:08:00.214353 139977740461824 logging_writer.py:48] [42500] global_step=42500, grad_norm=0.5, loss=1.14311 +I0912 20:08:00.222933 140000775775424 submission.py:307] 42500) loss = 1.143, grad_norm = 0.500 +I0912 20:12:17.094392 139977832781568 logging_writer.py:48] [43000] global_step=43000, grad_norm=0.5, loss=1.11903 +I0912 20:12:17.099488 140000775775424 submission.py:307] 43000) loss = 1.119, grad_norm = 0.500 +I0912 20:18:55.848701 139977740461824 logging_writer.py:48] [43500] global_step=43500, grad_norm=0.5, loss=1.09288 +I0912 20:18:55.854842 140000775775424 submission.py:307] 43500) loss = 1.093, grad_norm = 0.500 +I0912 20:24:48.797559 139977832781568 logging_writer.py:48] [44000] global_step=44000, grad_norm=0.5, loss=1.03837 +I0912 20:24:48.803272 140000775775424 submission.py:307] 44000) loss = 1.038, grad_norm = 0.500 +I0912 20:29:15.029472 139977740461824 logging_writer.py:48] [44500] global_step=44500, grad_norm=0.5, loss=1.15076 +I0912 20:29:15.035613 140000775775424 submission.py:307] 44500) loss = 1.151, grad_norm = 0.500 +I0912 20:37:18.456120 139977832781568 logging_writer.py:48] [45000] global_step=45000, grad_norm=0.5, loss=1.16221 +I0912 20:37:18.461033 140000775775424 submission.py:307] 45000) loss = 1.162, grad_norm = 0.500 +I0912 20:39:28.409649 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 20:40:13.635707 140000775775424 spec.py:346] Evaluating on the validation split. +I0912 20:41:21.026120 140000775775424 spec.py:363] Evaluating on the test split. +I0912 20:41:22.186751 140000775775424 submission_runner.py:516] Time since start: 34307.36s, Step: 45304, {'train/accuracy': 0.7406927614795918, 'train/loss': 3.3303010901626275, 'validation/accuracy': 0.59918, 'validation/loss': 3.6826671875, 'validation/num_examples': 50000, 'test/accuracy': 0.5385, 'test/loss': 2.3156078125, 'test/num_examples': 10000, 'score': 32065.929864406586, 'total_duration': 34307.3638010025, 'accumulated_submission_time': 32065.929864406586, 'accumulated_eval_time': 2123.4460899829865, 'accumulated_logging_time': 4.944806337356567} +I0912 20:41:22.619170 139977824388864 logging_writer.py:48] [45304] accumulated_eval_time=2123.45, accumulated_logging_time=4.94481, accumulated_submission_time=32065.9, global_step=45304, preemption_count=0, score=32065.9, test/accuracy=0.5385, test/loss=2.31561, test/num_examples=10000, total_duration=34307.4, train/accuracy=0.740693, train/loss=3.3303, validation/accuracy=0.59918, validation/loss=3.68267, validation/num_examples=50000 +I0912 20:42:53.409019 139977790818048 logging_writer.py:48] [45500] global_step=45500, grad_norm=0.5, loss=1.06046 +I0912 20:42:53.416018 140000775775424 submission.py:307] 45500) loss = 1.060, grad_norm = 0.500 +I0912 20:49:54.633343 139977824388864 logging_writer.py:48] [46000] global_step=46000, grad_norm=0.5, loss=1.08674 +I0912 20:49:54.639168 140000775775424 submission.py:307] 46000) loss = 1.087, grad_norm = 0.500 +I0912 20:56:04.911655 139977790818048 logging_writer.py:48] [46500] global_step=46500, grad_norm=0.5, loss=1.13727 +I0912 20:56:04.917118 140000775775424 submission.py:307] 46500) loss = 1.137, grad_norm = 0.500 +I0912 21:00:27.712263 139977824388864 logging_writer.py:48] [47000] global_step=47000, grad_norm=0.5, loss=1.07242 +I0912 21:00:27.719282 140000775775424 submission.py:307] 47000) loss = 1.072, grad_norm = 0.500 +I0912 21:08:39.006733 139977790818048 logging_writer.py:48] [47500] global_step=47500, grad_norm=0.5, loss=1.16482 +I0912 21:08:39.010653 140000775775424 submission.py:307] 47500) loss = 1.165, grad_norm = 0.500 +I0912 21:12:12.185183 139977824388864 logging_writer.py:48] [48000] global_step=48000, grad_norm=0.5, loss=1.02686 +I0912 21:12:12.189825 140000775775424 submission.py:307] 48000) loss = 1.027, grad_norm = 0.500 +I0912 21:14:39.705406 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 21:15:35.165803 140000775775424 spec.py:346] Evaluating on the validation split. +I0912 21:16:43.608438 140000775775424 spec.py:363] Evaluating on the test split. +I0912 21:16:44.674572 140000775775424 submission_runner.py:516] Time since start: 36429.85s, Step: 48241, {'train/accuracy': 0.7526307397959183, 'train/loss': 3.0826932946029975, 'validation/accuracy': 0.6075, 'validation/loss': 3.48755375, 'validation/num_examples': 50000, 'test/accuracy': 0.5486, 'test/loss': 2.253125, 'test/num_examples': 10000, 'score': 34056.41480588913, 'total_duration': 36429.851751089096, 'accumulated_submission_time': 34056.41480588913, 'accumulated_eval_time': 2248.4152448177338, 'accumulated_logging_time': 5.388753175735474} +I0912 21:16:44.997959 139977723676416 logging_writer.py:48] [48241] accumulated_eval_time=2248.42, accumulated_logging_time=5.38875, accumulated_submission_time=34056.4, global_step=48241, preemption_count=0, score=34056.4, test/accuracy=0.5486, test/loss=2.25312, test/num_examples=10000, total_duration=36429.9, train/accuracy=0.752631, train/loss=3.08269, validation/accuracy=0.6075, validation/loss=3.48755, validation/num_examples=50000 +I0912 21:19:50.992052 139977782425344 logging_writer.py:48] [48500] global_step=48500, grad_norm=0.5, loss=1.06412 +I0912 21:19:50.997551 140000775775424 submission.py:307] 48500) loss = 1.064, grad_norm = 0.500 +I0912 21:26:14.240628 139977723676416 logging_writer.py:48] [49000] global_step=49000, grad_norm=0.5, loss=0.982907 +I0912 21:26:14.259660 140000775775424 submission.py:307] 49000) loss = 0.983, grad_norm = 0.500 +I0912 21:30:55.390226 139977782425344 logging_writer.py:48] [49500] global_step=49500, grad_norm=0.5, loss=0.9992 +I0912 21:30:55.395956 140000775775424 submission.py:307] 49500) loss = 0.999, grad_norm = 0.500 +I0912 21:38:56.210087 139977723676416 logging_writer.py:48] [50000] global_step=50000, grad_norm=0.5, loss=1.09904 +I0912 21:38:56.213953 140000775775424 submission.py:307] 50000) loss = 1.099, grad_norm = 0.500 +I0912 21:42:32.934859 139977782425344 logging_writer.py:48] [50500] global_step=50500, grad_norm=0.499999, loss=0.933484 +I0912 21:42:32.943152 140000775775424 submission.py:307] 50500) loss = 0.933, grad_norm = 0.500 +I0912 21:48:51.441699 139977723676416 logging_writer.py:48] [51000] global_step=51000, grad_norm=0.5, loss=1.10099 +I0912 21:48:51.445698 140000775775424 submission.py:307] 51000) loss = 1.101, grad_norm = 0.500 +I0912 21:50:07.626468 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 21:50:47.261206 140000775775424 spec.py:346] Evaluating on the validation split. +I0912 21:51:59.545866 140000775775424 spec.py:363] Evaluating on the test split. +I0912 21:52:00.677315 140000775775424 submission_runner.py:516] Time since start: 38545.85s, Step: 51065, {'train/accuracy': 0.7146045918367347, 'train/loss': 4.287095050422513, 'validation/accuracy': 0.59812, 'validation/loss': 3.8641540625, 'validation/num_examples': 50000, 'test/accuracy': 0.551, 'test/loss': 2.254645703125, 'test/num_examples': 10000, 'score': 36049.56338047981, 'total_duration': 38545.85446166992, 'accumulated_submission_time': 36049.56338047981, 'accumulated_eval_time': 2361.465857744217, 'accumulated_logging_time': 5.720783233642578} +I0912 21:52:01.066843 139981372782336 logging_writer.py:48] [51065] accumulated_eval_time=2361.47, accumulated_logging_time=5.72078, accumulated_submission_time=36049.6, global_step=51065, preemption_count=0, score=36049.6, test/accuracy=0.551, test/loss=2.25465, test/num_examples=10000, total_duration=38545.9, train/accuracy=0.714605, train/loss=4.2871, validation/accuracy=0.59812, validation/loss=3.86415, validation/num_examples=50000 +I0912 21:56:21.033245 139977757247232 logging_writer.py:48] [51500] global_step=51500, grad_norm=0.5, loss=0.983306 +I0912 21:56:21.040655 140000775775424 submission.py:307] 51500) loss = 0.983, grad_norm = 0.500 +I0912 22:01:28.128259 139981372782336 logging_writer.py:48] [52000] global_step=52000, grad_norm=0.5, loss=1.00985 +I0912 22:01:28.132316 140000775775424 submission.py:307] 52000) loss = 1.010, grad_norm = 0.500 +I0912 22:09:21.184303 139977757247232 logging_writer.py:48] [52500] global_step=52500, grad_norm=0.5, loss=1.09602 +I0912 22:09:21.189157 140000775775424 submission.py:307] 52500) loss = 1.096, grad_norm = 0.500 +I0912 22:12:50.300360 139981372782336 logging_writer.py:48] [53000] global_step=53000, grad_norm=0.5, loss=0.994163 +I0912 22:12:50.306233 140000775775424 submission.py:307] 53000) loss = 0.994, grad_norm = 0.500 +I0912 22:19:06.608829 139977757247232 logging_writer.py:48] [53500] global_step=53500, grad_norm=0.5, loss=0.981594 +I0912 22:19:06.612667 140000775775424 submission.py:307] 53500) loss = 0.982, grad_norm = 0.500 +I0912 22:25:18.002866 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 22:25:59.027443 140000775775424 spec.py:346] Evaluating on the validation split. +I0912 22:27:06.062156 140000775775424 spec.py:363] Evaluating on the test split. +I0912 22:27:07.149380 140000775775424 submission_runner.py:516] Time since start: 40652.33s, Step: 53993, {'train/accuracy': 0.6931202168367347, 'train/loss': 4.989987743144133, 'validation/accuracy': 0.60402, 'validation/loss': 3.70380375, 'validation/num_examples': 50000, 'test/accuracy': 0.5426, 'test/loss': 2.3408603515625, 'test/num_examples': 10000, 'score': 38040.11103129387, 'total_duration': 40652.326677560806, 'accumulated_submission_time': 38040.11103129387, 'accumulated_eval_time': 2470.6195392608643, 'accumulated_logging_time': 6.12224555015564} +I0912 22:27:07.529785 139977706891008 logging_writer.py:48] [53993] accumulated_eval_time=2470.62, accumulated_logging_time=6.12225, accumulated_submission_time=38040.1, global_step=53993, preemption_count=0, score=38040.1, test/accuracy=0.5426, test/loss=2.34086, test/num_examples=10000, total_duration=40652.3, train/accuracy=0.69312, train/loss=4.98999, validation/accuracy=0.60402, validation/loss=3.7038, validation/num_examples=50000 +I0912 22:27:11.123795 139977748854528 logging_writer.py:48] [54000] global_step=54000, grad_norm=0.5, loss=1.07409 +I0912 22:27:11.127067 140000775775424 submission.py:307] 54000) loss = 1.074, grad_norm = 0.500 +I0912 22:32:14.428778 139977706891008 logging_writer.py:48] [54500] global_step=54500, grad_norm=0.5, loss=1.03183 +I0912 22:32:14.433471 140000775775424 submission.py:307] 54500) loss = 1.032, grad_norm = 0.500 +I0912 22:40:34.640470 139977748854528 logging_writer.py:48] [55000] global_step=55000, grad_norm=0.5, loss=1.07491 +I0912 22:40:34.645375 140000775775424 submission.py:307] 55000) loss = 1.075, grad_norm = 0.500 +I0912 22:44:16.141507 139977706891008 logging_writer.py:48] [55500] global_step=55500, grad_norm=0.5, loss=0.950578 +I0912 22:44:16.149398 140000775775424 submission.py:307] 55500) loss = 0.951, grad_norm = 0.500 +I0912 22:50:37.462334 139977748854528 logging_writer.py:48] [56000] global_step=56000, grad_norm=0.5, loss=0.965331 +I0912 22:50:37.466215 140000775775424 submission.py:307] 56000) loss = 0.965, grad_norm = 0.500 +I0912 22:56:46.725200 139977706891008 logging_writer.py:48] [56500] global_step=56500, grad_norm=0.499999, loss=0.923452 +I0912 22:56:46.733721 140000775775424 submission.py:307] 56500) loss = 0.923, grad_norm = 0.500 +I0912 23:00:25.570307 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 23:01:19.725131 140000775775424 spec.py:346] Evaluating on the validation split. +I0912 23:02:27.392586 140000775775424 spec.py:363] Evaluating on the test split. +I0912 23:02:28.513334 140000775775424 submission_runner.py:516] Time since start: 42773.69s, Step: 56938, {'train/accuracy': 0.7349131058673469, 'train/loss': 4.022158875757334, 'validation/accuracy': 0.60598, 'validation/loss': 3.7986165625, 'validation/num_examples': 50000, 'test/accuracy': 0.5461, 'test/loss': 2.324066796875, 'test/num_examples': 10000, 'score': 40032.12872648239, 'total_duration': 42773.69043254852, 'accumulated_submission_time': 40032.12872648239, 'accumulated_eval_time': 2593.562024831772, 'accumulated_logging_time': 6.515619516372681} +I0912 23:02:28.794502 139977748854528 logging_writer.py:48] [56938] accumulated_eval_time=2593.56, accumulated_logging_time=6.51562, accumulated_submission_time=40032.1, global_step=56938, preemption_count=0, score=40032.1, test/accuracy=0.5461, test/loss=2.32407, test/num_examples=10000, total_duration=42773.7, train/accuracy=0.734913, train/loss=4.02216, validation/accuracy=0.60598, validation/loss=3.79862, validation/num_examples=50000 +I0912 23:02:49.719710 139977757247232 logging_writer.py:48] [57000] global_step=57000, grad_norm=0.5, loss=1.02598 +I0912 23:02:49.724359 140000775775424 submission.py:307] 57000) loss = 1.026, grad_norm = 0.500 +I0912 23:10:42.430212 139977748854528 logging_writer.py:48] [57500] global_step=57500, grad_norm=0.5, loss=0.979313 +I0912 23:10:42.435361 140000775775424 submission.py:307] 57500) loss = 0.979, grad_norm = 0.500 +I0912 23:14:51.382319 139977757247232 logging_writer.py:48] [58000] global_step=58000, grad_norm=0.5, loss=0.916976 +I0912 23:14:51.386700 140000775775424 submission.py:307] 58000) loss = 0.917, grad_norm = 0.500 +I0912 23:21:10.922512 139977748854528 logging_writer.py:48] [58500] global_step=58500, grad_norm=0.5, loss=1.03138 +I0912 23:21:10.926496 140000775775424 submission.py:307] 58500) loss = 1.031, grad_norm = 0.500 +I0912 23:27:12.664704 139977757247232 logging_writer.py:48] [59000] global_step=59000, grad_norm=0.5, loss=0.968239 +I0912 23:27:12.668807 140000775775424 submission.py:307] 59000) loss = 0.968, grad_norm = 0.500 +I0912 23:31:35.604093 139977748854528 logging_writer.py:48] [59500] global_step=59500, grad_norm=0.5, loss=1.0205 +I0912 23:31:35.608190 140000775775424 submission.py:307] 59500) loss = 1.021, grad_norm = 0.500 +I0912 23:35:48.796160 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 23:36:32.487381 140000775775424 spec.py:346] Evaluating on the validation split. +I0912 23:37:52.501773 140000775775424 spec.py:363] Evaluating on the test split. +I0912 23:37:53.499625 140000775775424 submission_runner.py:516] Time since start: 44898.68s, Step: 59780, {'train/accuracy': 0.7512755102040817, 'train/loss': 3.751901743363361, 'validation/accuracy': 0.60982, 'validation/loss': 3.62400375, 'validation/num_examples': 50000, 'test/accuracy': 0.544, 'test/loss': 2.335040625, 'test/num_examples': 10000, 'score': 42026.04635977745, 'total_duration': 44898.67599797249, 'accumulated_submission_time': 42026.04635977745, 'accumulated_eval_time': 2718.264631986618, 'accumulated_logging_time': 6.805363655090332} +I0912 23:37:53.862483 139977757247232 logging_writer.py:48] [59780] accumulated_eval_time=2718.26, accumulated_logging_time=6.80536, accumulated_submission_time=42026, global_step=59780, preemption_count=0, score=42026, test/accuracy=0.544, test/loss=2.33504, test/num_examples=10000, total_duration=44898.7, train/accuracy=0.751276, train/loss=3.7519, validation/accuracy=0.60982, validation/loss=3.624, validation/num_examples=50000 +I0912 23:40:40.608777 139977832781568 logging_writer.py:48] [60000] global_step=60000, grad_norm=0.5, loss=1.01784 +I0912 23:40:40.615422 140000775775424 submission.py:307] 60000) loss = 1.018, grad_norm = 0.500 +I0912 23:44:56.553373 139977757247232 logging_writer.py:48] [60500] global_step=60500, grad_norm=0.5, loss=0.929709 +I0912 23:44:56.557208 140000775775424 submission.py:307] 60500) loss = 0.930, grad_norm = 0.500 +I0912 23:51:31.523284 139977832781568 logging_writer.py:48] [61000] global_step=61000, grad_norm=0.5, loss=0.883312 +I0912 23:51:31.527040 140000775775424 submission.py:307] 61000) loss = 0.883, grad_norm = 0.500 +I0912 23:57:35.893241 139977757247232 logging_writer.py:48] [61500] global_step=61500, grad_norm=0.5, loss=0.869233 +I0912 23:57:35.898113 140000775775424 submission.py:307] 61500) loss = 0.869, grad_norm = 0.500 +I0913 00:01:55.907558 139977832781568 logging_writer.py:48] [62000] global_step=62000, grad_norm=0.5, loss=0.978815 +I0913 00:01:55.911541 140000775775424 submission.py:307] 62000) loss = 0.979, grad_norm = 0.500 +I0913 00:10:01.386270 139977757247232 logging_writer.py:48] [62500] global_step=62500, grad_norm=0.5, loss=0.898184 +I0913 00:10:01.390040 140000775775424 submission.py:307] 62500) loss = 0.898, grad_norm = 0.500 +I0913 00:11:14.200325 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 00:11:50.966895 140000775775424 spec.py:346] Evaluating on the validation split. +I0913 00:12:58.398634 140000775775424 spec.py:363] Evaluating on the test split. +I0913 00:12:59.452566 140000775775424 submission_runner.py:516] Time since start: 47004.63s, Step: 62641, {'train/accuracy': 0.7533880739795918, 'train/loss': 3.9760337362484055, 'validation/accuracy': 0.5953, 'validation/loss': 4.1124975, 'validation/num_examples': 50000, 'test/accuracy': 0.5445, 'test/loss': 2.4248345703125, 'test/num_examples': 10000, 'score': 44017.4248752594, 'total_duration': 47004.629635334015, 'accumulated_submission_time': 44017.4248752594, 'accumulated_eval_time': 2823.5164263248444, 'accumulated_logging_time': 7.177429914474487} +I0913 00:12:59.804372 139977799210752 logging_writer.py:48] [62641] accumulated_eval_time=2823.52, accumulated_logging_time=7.17743, accumulated_submission_time=44017.4, global_step=62641, preemption_count=0, score=44017.4, test/accuracy=0.5445, test/loss=2.42483, test/num_examples=10000, total_duration=47004.6, train/accuracy=0.753388, train/loss=3.97603, validation/accuracy=0.5953, validation/loss=4.1125, validation/num_examples=50000 +I0913 00:15:56.360311 139965144938240 logging_writer.py:48] [63000] global_step=63000, grad_norm=0.5, loss=0.928835 +I0913 00:15:56.365406 140000775775424 submission.py:307] 63000) loss = 0.929, grad_norm = 0.500 +I0913 00:22:35.888167 139977799210752 logging_writer.py:48] [63500] global_step=63500, grad_norm=0.5, loss=0.938958 +I0913 00:22:35.892018 140000775775424 submission.py:307] 63500) loss = 0.939, grad_norm = 0.500 +I0913 00:29:05.031001 139965144938240 logging_writer.py:48] [64000] global_step=64000, grad_norm=0.5, loss=0.847355 +I0913 00:29:05.036792 140000775775424 submission.py:307] 64000) loss = 0.847, grad_norm = 0.500 +I0913 00:33:19.473836 139977799210752 logging_writer.py:48] [64500] global_step=64500, grad_norm=0.5, loss=0.902039 +I0913 00:33:19.480113 140000775775424 submission.py:307] 64500) loss = 0.902, grad_norm = 0.500 +I0913 00:41:27.202098 139965144938240 logging_writer.py:48] [65000] global_step=65000, grad_norm=0.5, loss=1.05563 +I0913 00:41:27.208215 140000775775424 submission.py:307] 65000) loss = 1.056, grad_norm = 0.500 +I0913 00:45:08.065379 139977799210752 logging_writer.py:48] [65500] global_step=65500, grad_norm=0.5, loss=0.9703 +I0913 00:45:08.071282 140000775775424 submission.py:307] 65500) loss = 0.970, grad_norm = 0.500 +I0913 00:46:20.594476 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 00:47:07.200145 140000775775424 spec.py:346] Evaluating on the validation split. +I0913 00:48:15.022639 140000775775424 spec.py:363] Evaluating on the test split. +I0913 00:48:16.010275 140000775775424 submission_runner.py:516] Time since start: 49121.19s, Step: 65621, {'train/accuracy': 0.7441804846938775, 'train/loss': 4.294895094268176, 'validation/accuracy': 0.60682, 'validation/loss': 3.9011953125, 'validation/num_examples': 50000, 'test/accuracy': 0.546, 'test/loss': 2.451838671875, 'test/num_examples': 10000, 'score': 46007.35127186775, 'total_duration': 49121.18770194054, 'accumulated_submission_time': 46007.35127186775, 'accumulated_eval_time': 2938.9320743083954, 'accumulated_logging_time': 7.538053512573242} +I0913 00:48:16.354599 139977715283712 logging_writer.py:48] [65621] accumulated_eval_time=2938.93, accumulated_logging_time=7.53805, accumulated_submission_time=46007.4, global_step=65621, preemption_count=0, score=46007.4, test/accuracy=0.546, test/loss=2.45184, test/num_examples=10000, total_duration=49121.2, train/accuracy=0.74418, train/loss=4.2949, validation/accuracy=0.60682, validation/loss=3.9012, validation/num_examples=50000 +I0913 00:53:09.572074 139981372782336 logging_writer.py:48] [66000] global_step=66000, grad_norm=0.5, loss=1.03735 +I0913 00:53:09.577845 140000775775424 submission.py:307] 66000) loss = 1.037, grad_norm = 0.500 +I0913 00:59:40.696490 139977715283712 logging_writer.py:48] [66500] global_step=66500, grad_norm=0.5, loss=0.822102 +I0913 00:59:40.714273 140000775775424 submission.py:307] 66500) loss = 0.822, grad_norm = 0.500 +I0913 01:03:57.718021 139981372782336 logging_writer.py:48] [67000] global_step=67000, grad_norm=0.5, loss=0.879154 +I0913 01:03:57.730172 140000775775424 submission.py:307] 67000) loss = 0.879, grad_norm = 0.500 +I0913 01:11:55.212043 139977715283712 logging_writer.py:48] [67500] global_step=67500, grad_norm=0.5, loss=0.832335 +I0913 01:11:55.217998 140000775775424 submission.py:307] 67500) loss = 0.832, grad_norm = 0.500 +I0913 01:15:39.817698 139981372782336 logging_writer.py:48] [68000] global_step=68000, grad_norm=0.5, loss=0.854876 +I0913 01:15:39.821733 140000775775424 submission.py:307] 68000) loss = 0.855, grad_norm = 0.500 +I0913 01:21:37.267016 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 01:22:23.091344 140000775775424 spec.py:346] Evaluating on the validation split. +I0913 01:23:44.917098 140000775775424 spec.py:363] Evaluating on the test split. +I0913 01:23:45.934671 140000775775424 submission_runner.py:516] Time since start: 51251.11s, Step: 68491, {'train/accuracy': 0.7308673469387755, 'train/loss': 4.70134127869898, 'validation/accuracy': 0.60286, 'validation/loss': 3.9544271875, 'validation/num_examples': 50000, 'test/accuracy': 0.5418, 'test/loss': 2.4661576171875, 'test/num_examples': 10000, 'score': 47999.289845228195, 'total_duration': 51251.1121070385, 'accumulated_submission_time': 47999.289845228195, 'accumulated_eval_time': 3067.5996243953705, 'accumulated_logging_time': 7.8918163776397705} +I0913 01:23:46.344092 139981372782336 logging_writer.py:48] [68491] accumulated_eval_time=3067.6, accumulated_logging_time=7.89182, accumulated_submission_time=47999.3, global_step=68491, preemption_count=0, score=47999.3, test/accuracy=0.5418, test/loss=2.46616, test/num_examples=10000, total_duration=51251.1, train/accuracy=0.730867, train/loss=4.70134, validation/accuracy=0.60286, validation/loss=3.95443, validation/num_examples=50000 +I0913 01:23:50.531060 139977740461824 logging_writer.py:48] [68500] global_step=68500, grad_norm=0.5, loss=0.895563 +I0913 01:23:50.535244 140000775775424 submission.py:307] 68500) loss = 0.896, grad_norm = 0.500 +I0913 01:29:24.206279 139981372782336 logging_writer.py:48] [69000] global_step=69000, grad_norm=0.5, loss=0.845775 +I0913 01:29:24.213917 140000775775424 submission.py:307] 69000) loss = 0.846, grad_norm = 0.500 +I0913 01:34:20.056565 139977740461824 logging_writer.py:48] [69500] global_step=69500, grad_norm=0.5, loss=0.948744 +I0913 01:34:20.060950 140000775775424 submission.py:307] 69500) loss = 0.949, grad_norm = 0.500 +I0913 01:42:08.441695 139981372782336 logging_writer.py:48] [70000] global_step=70000, grad_norm=0.5, loss=0.919155 +I0913 01:42:08.446732 140000775775424 submission.py:307] 70000) loss = 0.919, grad_norm = 0.500 +I0913 01:45:53.725349 139977740461824 logging_writer.py:48] [70500] global_step=70500, grad_norm=0.5, loss=0.861036 +I0913 01:45:53.736145 140000775775424 submission.py:307] 70500) loss = 0.861, grad_norm = 0.500 +I0913 01:52:06.299637 139981372782336 logging_writer.py:48] [71000] global_step=71000, grad_norm=0.5, loss=0.869785 +I0913 01:52:06.304679 140000775775424 submission.py:307] 71000) loss = 0.870, grad_norm = 0.500 +I0913 01:57:13.981459 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 01:57:49.563395 140000775775424 spec.py:346] Evaluating on the validation split. +I0913 01:58:56.603223 140000775775424 spec.py:363] Evaluating on the test split. +I0913 01:58:57.582027 140000775775424 submission_runner.py:516] Time since start: 53362.76s, Step: 71308, {'train/accuracy': 0.7486049107142857, 'train/loss': 4.223421057876275, 'validation/accuracy': 0.5905, 'validation/loss': 4.304688125, 'validation/num_examples': 50000, 'test/accuracy': 0.5403, 'test/loss': 2.485997265625, 'test/num_examples': 10000, 'score': 49997.043336868286, 'total_duration': 53362.75940608978, 'accumulated_submission_time': 49997.043336868286, 'accumulated_eval_time': 3171.210006713867, 'accumulated_logging_time': 8.32323670387268} +I0913 01:58:57.925559 139977723676416 logging_writer.py:48] [71308] accumulated_eval_time=3171.21, accumulated_logging_time=8.32324, accumulated_submission_time=49997, global_step=71308, preemption_count=0, score=49997, test/accuracy=0.5403, test/loss=2.486, test/num_examples=10000, total_duration=53362.8, train/accuracy=0.748605, train/loss=4.22342, validation/accuracy=0.5905, validation/loss=4.30469, validation/num_examples=50000 +I0913 02:00:10.107468 139977715283712 logging_writer.py:48] [71500] global_step=71500, grad_norm=0.5, loss=0.783187 +I0913 02:00:10.111557 140000775775424 submission.py:307] 71500) loss = 0.783, grad_norm = 0.500 +I0913 02:05:20.460700 139977723676416 logging_writer.py:48] [72000] global_step=72000, grad_norm=0.5, loss=0.817897 +I0913 02:05:20.464778 140000775775424 submission.py:307] 72000) loss = 0.818, grad_norm = 0.500 +I0913 02:13:34.424981 139977715283712 logging_writer.py:48] [72500] global_step=72500, grad_norm=0.5, loss=0.886132 +I0913 02:13:34.429627 140000775775424 submission.py:307] 72500) loss = 0.886, grad_norm = 0.500 +I0913 02:17:23.823337 139977723676416 logging_writer.py:48] [73000] global_step=73000, grad_norm=0.5, loss=0.788032 +I0913 02:17:23.852191 140000775775424 submission.py:307] 73000) loss = 0.788, grad_norm = 0.500 +I0913 02:23:39.669979 139977715283712 logging_writer.py:48] [73500] global_step=73500, grad_norm=0.5, loss=0.85083 +I0913 02:23:39.675583 140000775775424 submission.py:307] 73500) loss = 0.851, grad_norm = 0.500 +I0913 02:30:01.359055 139977723676416 logging_writer.py:48] [74000] global_step=74000, grad_norm=0.5, loss=0.76729 +I0913 02:30:01.366202 140000775775424 submission.py:307] 74000) loss = 0.767, grad_norm = 0.500 +I0913 02:32:18.911372 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 02:33:05.666545 140000775775424 spec.py:346] Evaluating on the validation split. +I0913 02:34:13.306517 140000775775424 spec.py:363] Evaluating on the test split. +I0913 02:34:14.327202 140000775775424 submission_runner.py:516] Time since start: 55479.50s, Step: 74295, {'train/accuracy': 0.7805325255102041, 'train/loss': 3.9461757114955356, 'validation/accuracy': 0.60874, 'validation/loss': 3.9354934375, 'validation/num_examples': 50000, 'test/accuracy': 0.5505, 'test/loss': 2.4941072265625, 'test/num_examples': 10000, 'score': 51987.919023275375, 'total_duration': 55479.50464963913, 'accumulated_submission_time': 51987.919023275375, 'accumulated_eval_time': 3286.6257762908936, 'accumulated_logging_time': 8.68415379524231} +I0913 02:34:14.713402 139977698498304 logging_writer.py:48] [74295] accumulated_eval_time=3286.63, accumulated_logging_time=8.68415, accumulated_submission_time=51987.9, global_step=74295, preemption_count=0, score=51987.9, test/accuracy=0.5505, test/loss=2.49411, test/num_examples=10000, total_duration=55479.5, train/accuracy=0.780533, train/loss=3.94618, validation/accuracy=0.60874, validation/loss=3.93549, validation/num_examples=50000 +I0913 02:36:16.468868 139977757247232 logging_writer.py:48] [74500] global_step=74500, grad_norm=0.5, loss=0.913088 +I0913 02:36:16.473266 140000775775424 submission.py:307] 74500) loss = 0.913, grad_norm = 0.500 +I0913 02:44:23.578018 139977698498304 logging_writer.py:48] [75000] global_step=75000, grad_norm=0.5, loss=0.773218 +I0913 02:44:23.582363 140000775775424 submission.py:307] 75000) loss = 0.773, grad_norm = 0.500 +I0913 02:48:18.813770 139977757247232 logging_writer.py:48] [75500] global_step=75500, grad_norm=0.5, loss=0.842224 +I0913 02:48:18.819859 140000775775424 submission.py:307] 75500) loss = 0.842, grad_norm = 0.500 +I0913 02:54:25.863495 139977698498304 logging_writer.py:48] [76000] global_step=76000, grad_norm=0.5, loss=0.782979 +I0913 02:54:25.872312 140000775775424 submission.py:307] 76000) loss = 0.783, grad_norm = 0.500 +I0913 03:00:45.468243 139977757247232 logging_writer.py:48] [76500] global_step=76500, grad_norm=0.5, loss=0.760953 +I0913 03:00:45.495341 140000775775424 submission.py:307] 76500) loss = 0.761, grad_norm = 0.500 +I0913 03:04:56.288933 139977698498304 logging_writer.py:48] [77000] global_step=77000, grad_norm=0.5, loss=0.833703 +I0913 03:04:56.293898 140000775775424 submission.py:307] 77000) loss = 0.834, grad_norm = 0.500 +I0913 03:07:31.920144 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 03:08:21.490815 140000775775424 spec.py:346] Evaluating on the validation split. +I0913 03:09:39.255182 140000775775424 spec.py:363] Evaluating on the test split. +I0913 03:09:40.224300 140000775775424 submission_runner.py:516] Time since start: 57605.40s, Step: 77189, {'train/accuracy': 0.7568160076530612, 'train/loss': 4.696759905133929, 'validation/accuracy': 0.6149, 'validation/loss': 4.01059, 'validation/num_examples': 50000, 'test/accuracy': 0.5454, 'test/loss': 2.54879609375, 'test/num_examples': 10000, 'score': 53978.819489479065, 'total_duration': 57605.40187692642, 'accumulated_submission_time': 53978.819489479065, 'accumulated_eval_time': 3414.930118083954, 'accumulated_logging_time': 9.079579591751099} +I0913 03:09:40.576480 139981372782336 logging_writer.py:48] [77189] accumulated_eval_time=3414.93, accumulated_logging_time=9.07958, accumulated_submission_time=53978.8, global_step=77189, preemption_count=0, score=53978.8, test/accuracy=0.5454, test/loss=2.5488, test/num_examples=10000, total_duration=57605.4, train/accuracy=0.756816, train/loss=4.69676, validation/accuracy=0.6149, validation/loss=4.01059, validation/num_examples=50000 +I0913 03:13:53.057813 139965144938240 logging_writer.py:48] [77500] global_step=77500, grad_norm=0.5, loss=0.916613 +I0913 03:13:53.065736 140000775775424 submission.py:307] 77500) loss = 0.917, grad_norm = 0.500 +I0913 03:18:15.925543 139981372782336 logging_writer.py:48] [78000] global_step=78000, grad_norm=0.5, loss=0.745283 +I0913 03:18:15.929738 140000775775424 submission.py:307] 78000) loss = 0.745, grad_norm = 0.500 +I0913 03:24:32.684141 139965144938240 logging_writer.py:48] [78500] global_step=78500, grad_norm=0.5, loss=0.732007 +I0913 03:24:32.688148 140000775775424 submission.py:307] 78500) loss = 0.732, grad_norm = 0.500 +I0913 03:30:51.234443 139981372782336 logging_writer.py:48] [79000] global_step=79000, grad_norm=0.5, loss=0.79025 +I0913 03:30:51.263784 140000775775424 submission.py:307] 79000) loss = 0.790, grad_norm = 0.500 +I0913 03:35:00.547186 139965144938240 logging_writer.py:48] [79500] global_step=79500, grad_norm=0.5, loss=0.8016 +I0913 03:35:00.552347 140000775775424 submission.py:307] 79500) loss = 0.802, grad_norm = 0.500 +I0913 03:42:55.137576 139981372782336 logging_writer.py:48] [80000] global_step=80000, grad_norm=0.5, loss=0.848419 +I0913 03:42:55.144645 140000775775424 submission.py:307] 80000) loss = 0.848, grad_norm = 0.500 +I0913 03:43:01.241533 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 03:43:39.682132 140000775775424 spec.py:346] Evaluating on the validation split. +I0913 03:44:48.085533 140000775775424 spec.py:363] Evaluating on the test split. +I0913 03:44:49.039283 140000775775424 submission_runner.py:516] Time since start: 59714.22s, Step: 80005, {'train/accuracy': 0.7893415178571429, 'train/loss': 3.8998624840561225, 'validation/accuracy': 0.5997, 'validation/loss': 4.322921875, 'validation/num_examples': 50000, 'test/accuracy': 0.5462, 'test/loss': 2.54585859375, 'test/num_examples': 10000, 'score': 55970.49907040596, 'total_duration': 59714.21665263176, 'accumulated_submission_time': 55970.49907040596, 'accumulated_eval_time': 3522.727690935135, 'accumulated_logging_time': 9.445403337478638} +I0913 03:44:49.351073 139977748854528 logging_writer.py:48] [80005] accumulated_eval_time=3522.73, accumulated_logging_time=9.4454, accumulated_submission_time=55970.5, global_step=80005, preemption_count=0, score=55970.5, test/accuracy=0.5462, test/loss=2.54586, test/num_examples=10000, total_duration=59714.2, train/accuracy=0.789342, train/loss=3.89986, validation/accuracy=0.5997, validation/loss=4.32292, validation/num_examples=50000 +I0913 03:49:05.641778 139977698498304 logging_writer.py:48] [80500] global_step=80500, grad_norm=0.5, loss=0.738146 +I0913 03:49:05.646796 140000775775424 submission.py:307] 80500) loss = 0.738, grad_norm = 0.500 +I0913 03:55:38.135170 139977748854528 logging_writer.py:48] [81000] global_step=81000, grad_norm=0.5, loss=0.795125 +I0913 03:55:38.141772 140000775775424 submission.py:307] 81000) loss = 0.795, grad_norm = 0.500 +I0913 04:02:00.777566 139977698498304 logging_writer.py:48] [81500] global_step=81500, grad_norm=0.5, loss=0.755148 +I0913 04:02:00.795143 140000775775424 submission.py:307] 81500) loss = 0.755, grad_norm = 0.500 +I0913 04:06:11.434671 139977748854528 logging_writer.py:48] [82000] global_step=82000, grad_norm=0.5, loss=0.839809 +I0913 04:06:11.439753 140000775775424 submission.py:307] 82000) loss = 0.840, grad_norm = 0.500 +I0913 04:14:29.823797 139977698498304 logging_writer.py:48] [82500] global_step=82500, grad_norm=0.5, loss=0.740099 +I0913 04:14:29.828452 140000775775424 submission.py:307] 82500) loss = 0.740, grad_norm = 0.500 +I0913 04:18:10.110389 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 04:18:50.390659 140000775775424 spec.py:346] Evaluating on the validation split. +I0913 04:19:58.305800 140000775775424 spec.py:363] Evaluating on the test split. +I0913 04:19:59.433410 140000775775424 submission_runner.py:516] Time since start: 61824.61s, Step: 82971, {'train/accuracy': 0.7889827806122449, 'train/loss': 4.016093974210778, 'validation/accuracy': 0.60164, 'validation/loss': 4.40888375, 'validation/num_examples': 50000, 'test/accuracy': 0.5315, 'test/loss': 2.653146484375, 'test/num_examples': 10000, 'score': 57962.18090510368, 'total_duration': 61824.61091375351, 'accumulated_submission_time': 57962.18090510368, 'accumulated_eval_time': 3632.050837993622, 'accumulated_logging_time': 9.765913724899292} +I0913 04:19:59.779326 139977832781568 logging_writer.py:48] [82971] accumulated_eval_time=3632.05, accumulated_logging_time=9.76591, accumulated_submission_time=57962.2, global_step=82971, preemption_count=0, score=57962.2, test/accuracy=0.5315, test/loss=2.65315, test/num_examples=10000, total_duration=61824.6, train/accuracy=0.788983, train/loss=4.01609, validation/accuracy=0.60164, validation/loss=4.40888, validation/num_examples=50000 +I0913 04:20:11.834341 139977732069120 logging_writer.py:48] [83000] global_step=83000, grad_norm=0.5, loss=0.718881 +I0913 04:20:11.837844 140000775775424 submission.py:307] 83000) loss = 0.719, grad_norm = 0.500 +I0913 04:26:40.242443 139977832781568 logging_writer.py:48] [83500] global_step=83500, grad_norm=0.5, loss=0.803197 +I0913 04:26:40.246356 140000775775424 submission.py:307] 83500) loss = 0.803, grad_norm = 0.500 +I0913 04:33:09.199049 139977732069120 logging_writer.py:48] [84000] global_step=84000, grad_norm=0.499999, loss=0.767785 +I0913 04:33:09.207163 140000775775424 submission.py:307] 84000) loss = 0.768, grad_norm = 0.500 +I0913 04:37:19.928745 139977832781568 logging_writer.py:48] [84500] global_step=84500, grad_norm=0.5, loss=0.74794 +I0913 04:37:19.934039 140000775775424 submission.py:307] 84500) loss = 0.748, grad_norm = 0.500 +I0913 04:45:22.230025 139977732069120 logging_writer.py:48] [85000] global_step=85000, grad_norm=0.5, loss=0.740031 +I0913 04:45:22.235213 140000775775424 submission.py:307] 85000) loss = 0.740, grad_norm = 0.500 +I0913 04:49:06.159022 139977832781568 logging_writer.py:48] [85500] global_step=85500, grad_norm=0.5, loss=0.762469 +I0913 04:49:06.162990 140000775775424 submission.py:307] 85500) loss = 0.762, grad_norm = 0.500 +I0913 04:53:20.396013 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 04:54:13.328716 140000775775424 spec.py:346] Evaluating on the validation split. +I0913 04:55:25.281908 140000775775424 spec.py:363] Evaluating on the test split. +I0913 04:55:26.223115 140000775775424 submission_runner.py:516] Time since start: 63951.40s, Step: 85892, {'train/accuracy': 0.7917928890306123, 'train/loss': 4.0056451291454085, 'validation/accuracy': 0.60622, 'validation/loss': 4.4052428125, 'validation/num_examples': 50000, 'test/accuracy': 0.5408, 'test/loss': 2.61608125, 'test/num_examples': 10000, 'score': 59955.80336523056, 'total_duration': 63951.40026164055, 'accumulated_submission_time': 59955.80336523056, 'accumulated_eval_time': 3757.8778371810913, 'accumulated_logging_time': 10.12097978591919} +I0913 04:55:26.595241 139977715283712 logging_writer.py:48] [85892] accumulated_eval_time=3757.88, accumulated_logging_time=10.121, accumulated_submission_time=59955.8, global_step=85892, preemption_count=0, score=59955.8, test/accuracy=0.5408, test/loss=2.61608, test/num_examples=10000, total_duration=63951.4, train/accuracy=0.791793, train/loss=4.00565, validation/accuracy=0.60622, validation/loss=4.40524, validation/num_examples=50000 +I0913 04:56:10.125471 139965144938240 logging_writer.py:48] [86000] global_step=86000, grad_norm=0.5, loss=0.817747 +I0913 04:56:10.128568 140000775775424 submission.py:307] 86000) loss = 0.818, grad_norm = 0.500 +I0913 05:02:50.547089 139977715283712 logging_writer.py:48] [86500] global_step=86500, grad_norm=0.5, loss=0.697027 +I0913 05:02:50.552119 140000775775424 submission.py:307] 86500) loss = 0.697, grad_norm = 0.500 +I0913 05:07:38.245735 139965144938240 logging_writer.py:48] [87000] global_step=87000, grad_norm=0.5, loss=0.78917 +I0913 05:07:38.251219 140000775775424 submission.py:307] 87000) loss = 0.789, grad_norm = 0.500 +I0913 05:15:23.313135 139977715283712 logging_writer.py:48] [87500] global_step=87500, grad_norm=0.5, loss=0.723695 +I0913 05:15:23.318255 140000775775424 submission.py:307] 87500) loss = 0.724, grad_norm = 0.500 +I0913 05:19:09.922038 139965144938240 logging_writer.py:48] [88000] global_step=88000, grad_norm=0.499999, loss=0.738832 +I0913 05:19:09.927072 140000775775424 submission.py:307] 88000) loss = 0.739, grad_norm = 0.500 +I0913 05:25:12.368015 139977715283712 logging_writer.py:48] [88500] global_step=88500, grad_norm=0.5, loss=0.786888 +I0913 05:25:12.373621 140000775775424 submission.py:307] 88500) loss = 0.787, grad_norm = 0.500 +I0913 05:28:52.178941 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 05:29:29.792267 140000775775424 spec.py:346] Evaluating on the validation split. +I0913 05:30:54.588155 140000775775424 spec.py:363] Evaluating on the test split. +I0913 05:30:55.531375 140000775775424 submission_runner.py:516] Time since start: 66080.71s, Step: 88710, {'train/accuracy': 0.7699697066326531, 'train/loss': 4.780690407266422, 'validation/accuracy': 0.60596, 'validation/loss': 4.3524725, 'validation/num_examples': 50000, 'test/accuracy': 0.5423, 'test/loss': 2.61897421875, 'test/num_examples': 10000, 'score': 61952.93928551674, 'total_duration': 66080.70891070366, 'accumulated_submission_time': 61952.93928551674, 'accumulated_eval_time': 3881.2304060459137, 'accumulated_logging_time': 10.505954504013062} +I0913 05:30:55.884940 139977841174272 logging_writer.py:48] [88710] accumulated_eval_time=3881.23, accumulated_logging_time=10.506, accumulated_submission_time=61952.9, global_step=88710, preemption_count=0, score=61952.9, test/accuracy=0.5423, test/loss=2.61897, test/num_examples=10000, total_duration=66080.7, train/accuracy=0.76997, train/loss=4.78069, validation/accuracy=0.60596, validation/loss=4.35247, validation/num_examples=50000 +I0913 05:33:12.268613 139977732069120 logging_writer.py:48] [89000] global_step=89000, grad_norm=0.5, loss=0.70986 +I0913 05:33:12.283486 140000775775424 submission.py:307] 89000) loss = 0.710, grad_norm = 0.500 +I0913 05:38:02.196398 139977841174272 logging_writer.py:48] [89500] global_step=89500, grad_norm=0.5, loss=0.777562 +I0913 05:38:02.200525 140000775775424 submission.py:307] 89500) loss = 0.778, grad_norm = 0.500 +I0913 05:46:03.207736 139977732069120 logging_writer.py:48] [90000] global_step=90000, grad_norm=0.5, loss=0.823879 +I0913 05:46:03.211688 140000775775424 submission.py:307] 90000) loss = 0.824, grad_norm = 0.500 +I0913 05:49:55.428700 139977841174272 logging_writer.py:48] [90500] global_step=90500, grad_norm=0.5, loss=0.719666 +I0913 05:49:55.433883 140000775775424 submission.py:307] 90500) loss = 0.720, grad_norm = 0.500 +I0913 05:56:04.075578 139977732069120 logging_writer.py:48] [91000] global_step=91000, grad_norm=0.5, loss=0.873964 +I0913 05:56:04.080886 140000775775424 submission.py:307] 91000) loss = 0.874, grad_norm = 0.500 +I0913 06:02:30.601145 139977841174272 logging_writer.py:48] [91500] global_step=91500, grad_norm=0.5, loss=0.744396 +I0913 06:02:30.626551 140000775775424 submission.py:307] 91500) loss = 0.744, grad_norm = 0.500 +I0913 06:04:17.191857 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 06:05:08.757254 140000775775424 spec.py:346] Evaluating on the validation split. +I0913 06:06:16.001237 140000775775424 spec.py:363] Evaluating on the test split. +I0913 06:06:16.949918 140000775775424 submission_runner.py:516] Time since start: 68202.13s, Step: 91743, {'train/accuracy': 0.7998046875, 'train/loss': 4.096698683135363, 'validation/accuracy': 0.59458, 'validation/loss': 4.7257515625, 'validation/num_examples': 50000, 'test/accuracy': 0.5388, 'test/loss': 2.69848046875, 'test/num_examples': 10000, 'score': 63943.923169612885, 'total_duration': 68202.12734985352, 'accumulated_submission_time': 63943.923169612885, 'accumulated_eval_time': 4000.9886140823364, 'accumulated_logging_time': 10.871410131454468} +I0913 06:06:17.308762 139977799210752 logging_writer.py:48] [91743] accumulated_eval_time=4000.99, accumulated_logging_time=10.8714, accumulated_submission_time=63943.9, global_step=91743, preemption_count=0, score=63943.9, test/accuracy=0.5388, test/loss=2.69848, test/num_examples=10000, total_duration=68202.1, train/accuracy=0.799805, train/loss=4.0967, validation/accuracy=0.59458, validation/loss=4.72575, validation/num_examples=50000 +I0913 06:08:54.678261 139981372782336 logging_writer.py:48] [92000] global_step=92000, grad_norm=0.5, loss=0.714592 +I0913 06:08:54.683308 140000775775424 submission.py:307] 92000) loss = 0.715, grad_norm = 0.500 +I0913 06:17:00.779650 139977799210752 logging_writer.py:48] [92500] global_step=92500, grad_norm=0.5, loss=0.680729 +I0913 06:17:00.783483 140000775775424 submission.py:307] 92500) loss = 0.681, grad_norm = 0.500 +I0913 06:21:04.311223 139981372782336 logging_writer.py:48] [93000] global_step=93000, grad_norm=0.5, loss=0.714446 +I0913 06:21:04.317244 140000775775424 submission.py:307] 93000) loss = 0.714, grad_norm = 0.500 +I0913 06:27:10.365570 139977799210752 logging_writer.py:48] [93500] global_step=93500, grad_norm=0.5, loss=0.729626 +I0913 06:27:10.370434 140000775775424 submission.py:307] 93500) loss = 0.730, grad_norm = 0.500 +I0913 06:33:34.546448 139981372782336 logging_writer.py:48] [94000] global_step=94000, grad_norm=0.5, loss=0.69949 +I0913 06:33:34.567515 140000775775424 submission.py:307] 94000) loss = 0.699, grad_norm = 0.500 +I0913 06:37:44.171612 139977799210752 logging_writer.py:48] [94500] global_step=94500, grad_norm=0.5, loss=0.695173 +I0913 06:37:44.177286 140000775775424 submission.py:307] 94500) loss = 0.695, grad_norm = 0.500 +I0913 06:39:39.906518 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 06:40:32.791809 140000775775424 spec.py:346] Evaluating on the validation split. +I0913 06:41:45.024685 140000775775424 spec.py:363] Evaluating on the test split. +I0913 06:41:45.952360 140000775775424 submission_runner.py:516] Time since start: 70331.13s, Step: 94645, {'train/accuracy': 0.7890824298469388, 'train/loss': 4.5770973672672195, 'validation/accuracy': 0.61568, 'validation/loss': 4.1705865625, 'validation/num_examples': 50000, 'test/accuracy': 0.5426, 'test/loss': 2.6647490234375, 'test/num_examples': 10000, 'score': 65940.45378279686, 'total_duration': 70331.12963294983, 'accumulated_submission_time': 65940.45378279686, 'accumulated_eval_time': 4127.034431934357, 'accumulated_logging_time': 11.24164605140686} +I0913 06:41:46.391641 139977849566976 logging_writer.py:48] [94645] accumulated_eval_time=4127.03, accumulated_logging_time=11.2416, accumulated_submission_time=65940.5, global_step=94645, preemption_count=0, score=65940.5, test/accuracy=0.5426, test/loss=2.66475, test/num_examples=10000, total_duration=70331.1, train/accuracy=0.789082, train/loss=4.5771, validation/accuracy=0.61568, validation/loss=4.17059, validation/num_examples=50000 +I0913 06:46:42.812168 139977815996160 logging_writer.py:48] [95000] global_step=95000, grad_norm=0.5, loss=0.75794 +I0913 06:46:42.817381 140000775775424 submission.py:307] 95000) loss = 0.758, grad_norm = 0.500 +I0913 06:51:20.623584 139977849566976 logging_writer.py:48] [95500] global_step=95500, grad_norm=0.5, loss=0.691851 +I0913 06:51:20.627477 140000775775424 submission.py:307] 95500) loss = 0.692, grad_norm = 0.500 +I0913 06:57:27.721843 139977815996160 logging_writer.py:48] [96000] global_step=96000, grad_norm=0.5, loss=0.639298 +I0913 06:57:27.727766 140000775775424 submission.py:307] 96000) loss = 0.639, grad_norm = 0.500 +I0913 07:03:56.080560 139977849566976 logging_writer.py:48] [96500] global_step=96500, grad_norm=0.5, loss=0.651977 +I0913 07:03:56.088548 140000775775424 submission.py:307] 96500) loss = 0.652, grad_norm = 0.500 +I0913 07:08:07.584525 139977815996160 logging_writer.py:48] [97000] global_step=97000, grad_norm=0.5, loss=0.783772 +I0913 07:08:07.589147 140000775775424 submission.py:307] 97000) loss = 0.784, grad_norm = 0.500 +I0913 07:15:06.957711 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 07:15:45.888704 140000775775424 spec.py:346] Evaluating on the validation split. +I0913 07:17:09.127393 140000775775424 spec.py:363] Evaluating on the test split. +I0913 07:17:10.076625 140000775775424 submission_runner.py:516] Time since start: 72455.25s, Step: 97447, {'train/accuracy': 0.8005221619897959, 'train/loss': 4.370849609375, 'validation/accuracy': 0.59172, 'validation/loss': 5.0194925, 'validation/num_examples': 50000, 'test/accuracy': 0.5414, 'test/loss': 2.73567109375, 'test/num_examples': 10000, 'score': 67931.78388285637, 'total_duration': 72455.25379180908, 'accumulated_submission_time': 67931.78388285637, 'accumulated_eval_time': 4250.153058290482, 'accumulated_logging_time': 11.692513704299927} +I0913 07:17:10.476279 139977774032640 logging_writer.py:48] [97447] accumulated_eval_time=4250.15, accumulated_logging_time=11.6925, accumulated_submission_time=67931.8, global_step=97447, preemption_count=0, score=67931.8, test/accuracy=0.5414, test/loss=2.73567, test/num_examples=10000, total_duration=72455.3, train/accuracy=0.800522, train/loss=4.37085, validation/accuracy=0.59172, validation/loss=5.01949, validation/num_examples=50000 +I0913 07:17:26.278968 139977732069120 logging_writer.py:48] [97500] global_step=97500, grad_norm=0.5, loss=0.726028 +I0913 07:17:26.283252 140000775775424 submission.py:307] 97500) loss = 0.726, grad_norm = 0.500 +I0913 07:21:46.618813 139977774032640 logging_writer.py:48] [98000] global_step=98000, grad_norm=0.5, loss=0.637176 +I0913 07:21:46.623472 140000775775424 submission.py:307] 98000) loss = 0.637, grad_norm = 0.500 +I0913 07:28:18.708159 139977732069120 logging_writer.py:48] [98500] global_step=98500, grad_norm=0.5, loss=0.66364 +I0913 07:28:18.712160 140000775775424 submission.py:307] 98500) loss = 0.664, grad_norm = 0.500 +I0913 07:34:46.487302 139977774032640 logging_writer.py:48] [99000] global_step=99000, grad_norm=0.5, loss=0.610662 +I0913 07:34:46.498198 140000775775424 submission.py:307] 99000) loss = 0.611, grad_norm = 0.500 +I0913 07:38:56.351129 139977732069120 logging_writer.py:48] [99500] global_step=99500, grad_norm=0.5, loss=0.618113 +I0913 07:38:56.357746 140000775775424 submission.py:307] 99500) loss = 0.618, grad_norm = 0.500 +I0913 07:47:04.727615 139977774032640 logging_writer.py:48] [100000] global_step=100000, grad_norm=0.5, loss=0.788582 +I0913 07:47:04.731643 140000775775424 submission.py:307] 100000) loss = 0.789, grad_norm = 0.500 +I0913 07:50:27.594517 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 07:51:17.768687 140000775775424 spec.py:346] Evaluating on the validation split. +I0913 07:52:24.730157 140000775775424 spec.py:363] Evaluating on the test split. +I0913 07:52:25.978285 140000775775424 submission_runner.py:516] Time since start: 74571.16s, Step: 100449, {'train/accuracy': 0.8359574298469388, 'train/loss': 3.3949999128069197, 'validation/accuracy': 0.59442, 'validation/loss': 4.782189375, 'validation/num_examples': 50000, 'test/accuracy': 0.5364, 'test/loss': 2.7882865234375, 'test/num_examples': 10000, 'score': 69922.41740727425, 'total_duration': 74571.15584874153, 'accumulated_submission_time': 69922.41740727425, 'accumulated_eval_time': 4368.536885738373, 'accumulated_logging_time': 12.103090524673462} +I0913 07:52:26.317004 139977723676416 logging_writer.py:48] [100449] accumulated_eval_time=4368.54, accumulated_logging_time=12.1031, accumulated_submission_time=69922.4, global_step=100449, preemption_count=0, score=69922.4, test/accuracy=0.5364, test/loss=2.78829, test/num_examples=10000, total_duration=74571.2, train/accuracy=0.835957, train/loss=3.395, validation/accuracy=0.59442, validation/loss=4.78219, validation/num_examples=50000 +I0913 07:52:44.267974 139977815996160 logging_writer.py:48] [100500] global_step=100500, grad_norm=0.5, loss=0.620102 +I0913 07:52:44.273667 140000775775424 submission.py:307] 100500) loss = 0.620, grad_norm = 0.500 +I0913 07:59:14.757636 139977723676416 logging_writer.py:48] [101000] global_step=101000, grad_norm=0.5, loss=0.690663 +I0913 07:59:14.762608 140000775775424 submission.py:307] 101000) loss = 0.691, grad_norm = 0.500 +I0913 08:05:55.621662 139977815996160 logging_writer.py:48] [101500] global_step=101500, grad_norm=0.5, loss=0.60999 +I0913 08:05:55.627659 140000775775424 submission.py:307] 101500) loss = 0.610, grad_norm = 0.500 +I0913 08:10:03.696566 139977723676416 logging_writer.py:48] [102000] global_step=102000, grad_norm=0.5, loss=0.701143 +I0913 08:10:03.702738 140000775775424 submission.py:307] 102000) loss = 0.701, grad_norm = 0.500 +I0913 08:18:04.925536 139977815996160 logging_writer.py:48] [102500] global_step=102500, grad_norm=0.5, loss=0.692208 +I0913 08:18:04.931221 140000775775424 submission.py:307] 102500) loss = 0.692, grad_norm = 0.500 +I0913 08:21:56.699437 139977723676416 logging_writer.py:48] [103000] global_step=103000, grad_norm=0.5, loss=0.662734 +I0913 08:21:56.703483 140000775775424 submission.py:307] 103000) loss = 0.663, grad_norm = 0.500 +I0913 08:25:52.559943 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 08:26:41.684019 140000775775424 spec.py:346] Evaluating on the validation split. +I0913 08:27:49.203456 140000775775424 spec.py:363] Evaluating on the test split. +I0913 08:27:50.359293 140000775775424 submission_runner.py:516] Time since start: 76695.54s, Step: 103364, {'train/accuracy': 0.8005022321428571, 'train/loss': 4.6372011145766905, 'validation/accuracy': 0.60828, 'validation/loss': 4.66229375, 'validation/num_examples': 50000, 'test/accuracy': 0.5409, 'test/loss': 2.81368828125, 'test/num_examples': 10000, 'score': 71921.09128928185, 'total_duration': 76695.53641319275, 'accumulated_submission_time': 71921.09128928185, 'accumulated_eval_time': 4486.3357853889465, 'accumulated_logging_time': 12.450941801071167} +I0913 08:27:50.754010 139977740461824 logging_writer.py:48] [103364] accumulated_eval_time=4486.34, accumulated_logging_time=12.4509, accumulated_submission_time=71921.1, global_step=103364, preemption_count=0, score=71921.1, test/accuracy=0.5409, test/loss=2.81369, test/num_examples=10000, total_duration=76695.5, train/accuracy=0.800502, train/loss=4.6372, validation/accuracy=0.60828, validation/loss=4.66229, validation/num_examples=50000 +I0913 08:28:59.599461 139981372782336 logging_writer.py:48] [103500] global_step=103500, grad_norm=0.5, loss=0.69253 +I0913 08:28:59.604003 140000775775424 submission.py:307] 103500) loss = 0.693, grad_norm = 0.500 +I0913 08:35:48.737771 139977740461824 logging_writer.py:48] [104000] global_step=104000, grad_norm=0.5, loss=0.658222 +I0913 08:35:48.746842 140000775775424 submission.py:307] 104000) loss = 0.658, grad_norm = 0.500 +I0913 08:40:26.814197 139981372782336 logging_writer.py:48] [104500] global_step=104500, grad_norm=0.5, loss=0.789505 +I0913 08:40:26.819159 140000775775424 submission.py:307] 104500) loss = 0.790, grad_norm = 0.500 +I0913 08:48:09.426779 139977740461824 logging_writer.py:48] [105000] global_step=105000, grad_norm=0.5, loss=0.670804 +I0913 08:48:09.432027 140000775775424 submission.py:307] 105000) loss = 0.671, grad_norm = 0.500 +I0913 08:52:07.422962 139981372782336 logging_writer.py:48] [105500] global_step=105500, grad_norm=0.5, loss=0.641659 +I0913 08:52:07.439069 140000775775424 submission.py:307] 105500) loss = 0.642, grad_norm = 0.500 +I0913 08:57:54.190269 139977740461824 logging_writer.py:48] [106000] global_step=106000, grad_norm=0.5, loss=0.653852 +I0913 08:57:54.194163 140000775775424 submission.py:307] 106000) loss = 0.654, grad_norm = 0.500 +I0913 09:01:14.792334 140000775775424 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 09:01:53.041376 140000775775424 spec.py:346] Evaluating on the validation split. +I0913 09:03:18.960484 140000775775424 spec.py:363] Evaluating on the test split. +I0913 09:03:19.981229 140000775775424 submission_runner.py:516] Time since start: 78825.16s, Step: 106184, {'train/accuracy': 0.7851363201530612, 'train/loss': 5.05701866928412, 'validation/accuracy': 0.61554, 'validation/loss': 4.266904375, 'validation/num_examples': 50000, 'test/accuracy': 0.5441, 'test/loss': 2.7782767578125, 'test/num_examples': 10000, 'score': 73916.09183645248, 'total_duration': 78825.1583442688, 'accumulated_submission_time': 73916.09183645248, 'accumulated_eval_time': 4611.524318695068, 'accumulated_logging_time': 12.854808330535889} +I0913 09:03:20.378318 139977824388864 logging_writer.py:48] [106184] accumulated_eval_time=4611.52, accumulated_logging_time=12.8548, accumulated_submission_time=73916.1, global_step=106184, preemption_count=0, score=73916.1, test/accuracy=0.5441, test/loss=2.77828, test/num_examples=10000, total_duration=78825.2, train/accuracy=0.785136, train/loss=5.05702, validation/accuracy=0.61554, validation/loss=4.2669, validation/num_examples=50000 +I0913 09:06:06.853802 139977715283712 logging_writer.py:48] [106500] global_step=106500, grad_norm=0.5, loss=0.552604 +I0913 09:06:06.857723 140000775775424 submission.py:307] 106500) loss = 0.553, grad_norm = 0.500 +I0913 09:10:56.014981 139977824388864 logging_writer.py:48] [107000] global_step=107000, grad_norm=0.5, loss=0.730043 +I0913 09:10:56.021245 140000775775424 submission.py:307] 107000) loss = 0.730, grad_norm = 0.500 +I0913 09:18:48.490345 139977715283712 logging_writer.py:48] [107500] global_step=107500, grad_norm=0.5, loss=0.711751 +I0913 09:18:48.494432 140000775775424 submission.py:307] 107500) loss = 0.712, grad_norm = 0.500 +I0913 09:22:44.633194 139977824388864 logging_writer.py:48] [108000] global_step=108000, grad_norm=0.5, loss=0.634528 +I0913 09:22:44.640312 140000775775424 submission.py:307] 108000) loss = 0.635, grad_norm = 0.500 +I0913 09:28:41.944233 139977715283712 logging_writer.py:48] [108500] global_step=108500, grad_norm=0.5, loss=0.590555 +I0913 09:28:41.949335 140000775775424 submission.py:307] 108500) loss = 0.591, grad_norm = 0.500 +I0913 09:35:24.181925 139977824388864 logging_writer.py:48] [109000] global_step=109000, grad_norm=0.5, loss=0.62977 +I0913 09:35:24.187888 140000775775424 submission.py:307] 109000) loss = 0.630, grad_norm = 0.500 +I0913 09:36:36.266973 139977715283712 logging_writer.py:48] [109186] global_step=109186, preemption_count=0, score=75906.5 +I0913 09:36:41.754626 140000775775424 submission_runner.py:857] Final imagenet_resnet score: 75906.49300265312 diff --git a/logs/self_tuning/ademamix_golden/study_1/imagenet_resnet_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_1/imagenet_resnet_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..2d4d8b5e --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/imagenet_resnet_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,39 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/accuracy,test/loss,test/num_examples,total_duration,train/accuracy,train/loss,validation/accuracy,validation/loss,validation/num_examples +211.23752093315125,0.0,184.3222272396088,1,0,184.3222272396088,0.0008,6.91190625,10000,396.2264976501465,0.0011360012755102,6.911248654735331,0.00026,6.914865,50000 +349.44077491760254,0.2667188644409179,2180.3810358047485,1811,0,2180.3810358047485,0.0544,5.50634375,10000,2534.2220618724823,0.0714883609693877,5.665470045440051,0.07132,5.70609875,50000 +472.9282770156861,0.3891248703002929,4173.907923221588,4341,0,4173.907923221588,0.1697,4.324318359375,10000,4655.071031808853,0.2274194834183673,4.783149252132493,0.2126,4.719998125,50000 +596.0737833976746,0.6317288875579834,6168.158279180527,7220,0,6168.158279180527,0.3084,3.45687734375,10000,6777.990676403046,0.3622249681122449,4.767920046436544,0.36892,3.9849725,50000 +703.6881942749023,0.8909380435943604,8158.115170717239,10186,0,8158.115170717239,0.3987,2.8301828125,10000,8883.059608697891,0.4917091836734694,3.873329318299586,0.46566,3.754048125,50000 +822.8465011119843,1.123784065246582,10148.827378749847,13198,0,10148.827378749847,0.4514,2.54749296875,10000,11003.156066417694,0.5372090242346939,4.049487055564414,0.50746,3.7652409375,50000 +937.401035785675,1.4320638179779053,12142.09969496727,16058,0,12142.09969496727,0.4753,2.3972625,10000,13120.20466184616,0.5820910395408163,3.633132311762596,0.53814,3.2271865625,50000 +1049.0939209461212,1.754276990890503,14133.677445173264,19027,0,14133.677445173264,0.4994,2.3539578125,10000,15230.481875181198,0.6128427933673469,3.5851770517777424,0.55534,3.4209021875,50000 +1173.4251527786255,2.109804153442383,16124.675315141678,21984,0,16124.675315141678,0.515,2.26014921875,10000,17352.515357732773,0.6188815369897959,3.934852132991869,0.5705,3.4900921875,50000 +1294.1361372470856,2.48134183883667,18115.26232433319,24826,0,18115.26232433319,0.5172,2.265676171875,10000,19474.4873392582,0.6424784757653061,3.722700780751754,0.58654,3.1755021875,50000 +1408.1605834960938,2.901373863220215,20106.77537536621,27803,0,20106.77537536621,0.5188,2.329924609375,10000,21586.175618886948,0.6767976721938775,3.371230144889988,0.582,3.53975875,50000 +1530.5059394836426,3.310522079467773,22101.098651885983,30744,0,22101.098651885983,0.5315,2.2064314453125,10000,23708.939203500748,0.682258450255102,3.395152812101403,0.597,3.24307375,50000 +1655.1419308185575,3.656352043151856,24099.83566331864,33587,0,24099.83566331864,0.5299,2.2689298828125,10000,25842.06825828552,0.6550542091836735,4.210914456114477,0.5883,3.418475625,50000 +1766.7947697639463,3.978273391723633,26091.93958926201,36561,0,26091.93958926201,0.5332,2.25507109375,10000,27953.09684515,0.6913066007653061,3.5630960269850127,0.5924,3.4600703125,50000 +1889.333539485932,4.304686069488525,28084.79198908806,39508,0,28084.79198908806,0.5406,2.2091263671875,10000,30074.523801088333,0.6902901785714286,4.021324313416773,0.58788,3.737465,50000 +2009.6694073677063,4.598609685897827,30076.157195091248,42336,0,30076.157195091248,0.5405,2.2322970703125,10000,32196.38384056092,0.6846500318877551,4.242830237563775,0.59668,3.5448959375,50000 +2123.4460899829865,4.944806337356567,32065.929864406586,45304,0,32065.929864406586,0.5385,2.3156078125,10000,34307.3638010025,0.7406927614795918,3.3303010901626275,0.59918,3.6826671875,50000 +2248.4152448177338,5.388753175735474,34056.41480588913,48241,0,34056.41480588913,0.5486,2.253125,10000,36429.851751089096,0.7526307397959183,3.082693294602997,0.6075,3.48755375,50000 +2361.465857744217,5.720783233642578,36049.56338047981,51065,0,36049.56338047981,0.551,2.254645703125,10000,38545.85446166992,0.7146045918367347,4.287095050422513,0.59812,3.8641540625,50000 +2470.6195392608643,6.12224555015564,38040.11103129387,53993,0,38040.11103129387,0.5426,2.3408603515625,10000,40652.326677560806,0.6931202168367347,4.989987743144133,0.60402,3.70380375,50000 +2593.562024831772,6.515619516372681,40032.12872648239,56938,0,40032.12872648239,0.5461,2.324066796875,10000,42773.69043254852,0.7349131058673469,4.022158875757334,0.60598,3.7986165625,50000 +2718.264631986618,6.805363655090332,42026.04635977745,59780,0,42026.04635977745,0.544,2.335040625,10000,44898.67599797249,0.7512755102040817,3.751901743363361,0.60982,3.62400375,50000 +2823.5164263248444,7.177429914474487,44017.4248752594,62641,0,44017.4248752594,0.5445,2.4248345703125,10000,47004.629635334015,0.7533880739795918,3.9760337362484055,0.5953,4.1124975,50000 +2938.9320743083954,7.538053512573242,46007.35127186775,65621,0,46007.35127186775,0.546,2.451838671875,10000,49121.18770194054,0.7441804846938775,4.294895094268176,0.60682,3.9011953125,50000 +3067.5996243953705,7.89181637763977,47999.2898452282,68491,0,47999.2898452282,0.5418,2.4661576171875,10000,51251.1121070385,0.7308673469387755,4.70134127869898,0.60286,3.9544271875,50000 +3171.210006713867,8.32323670387268,49997.043336868286,71308,0,49997.043336868286,0.5403,2.485997265625,10000,53362.75940608978,0.7486049107142857,4.223421057876275,0.5905,4.304688125,50000 +3286.6257762908936,8.68415379524231,51987.919023275375,74295,0,51987.919023275375,0.5505,2.4941072265625,10000,55479.50464963913,0.7805325255102041,3.946175711495536,0.60874,3.9354934375,50000 +3414.930118083954,9.0795795917511,53978.819489479065,77189,0,53978.819489479065,0.5454,2.54879609375,10000,57605.40187692642,0.7568160076530612,4.696759905133929,0.6149,4.01059,50000 +3522.727690935135,9.445403337478638,55970.49907040596,80005,0,55970.49907040596,0.5462,2.54585859375,10000,59714.21665263176,0.7893415178571429,3.8998624840561225,0.5997,4.322921875,50000 +3632.050837993622,9.765913724899292,57962.18090510368,82971,0,57962.18090510368,0.5315,2.653146484375,10000,61824.61091375351,0.7889827806122449,4.016093974210778,0.60164,4.40888375,50000 +3757.8778371810913,10.12097978591919,59955.80336523056,85892,0,59955.80336523056,0.5408,2.61608125,10000,63951.40026164055,0.7917928890306123,4.0056451291454085,0.60622,4.4052428125,50000 +3881.2304060459137,10.505954504013062,61952.93928551674,88710,0,61952.93928551674,0.5423,2.61897421875,10000,66080.70891070366,0.7699697066326531,4.780690407266422,0.60596,4.3524725,50000 +4000.988614082336,10.871410131454468,63943.923169612885,91743,0,63943.923169612885,0.5388,2.69848046875,10000,68202.12734985352,0.7998046875,4.096698683135363,0.59458,4.7257515625,50000 +4127.034431934357,11.24164605140686,65940.45378279686,94645,0,65940.45378279686,0.5426,2.6647490234375,10000,70331.12963294983,0.7890824298469388,4.57709736726722,0.61568,4.1705865625,50000 +4250.153058290482,11.692513704299929,67931.78388285637,97447,0,67931.78388285637,0.5414,2.73567109375,10000,72455.25379180908,0.8005221619897959,4.370849609375,0.59172,5.0194925,50000 +4368.536885738373,12.103090524673462,69922.41740727425,100449,0,69922.41740727425,0.5364,2.7882865234375,10000,74571.15584874153,0.8359574298469388,3.39499991280692,0.59442,4.782189375,50000 +4486.3357853889465,12.450941801071169,71921.09128928185,103364,0,71921.09128928185,0.5409,2.81368828125,10000,76695.53641319275,0.8005022321428571,4.6372011145766905,0.60828,4.66229375,50000 +4611.524318695068,12.854808330535889,73916.09183645248,106184,0,73916.09183645248,0.5441,2.7782767578125,10000,78825.1583442688,0.7851363201530612,5.05701866928412,0.61554,4.266904375,50000 diff --git a/logs/self_tuning/ademamix_golden/study_1/imagenet_resnet_pytorch/trial_1/events.out.tfevents.1789211374.c9d641ec7cfd.38.0.v2 b/logs/self_tuning/ademamix_golden/study_1/imagenet_resnet_pytorch/trial_1/events.out.tfevents.1789211374.c9d641ec7cfd.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..f1713bac53fbd165b639fb6ab302447594d46c87 GIT binary patch literal 107782 zcmbT9c|Z;88^^O(Lb6LFmA&jjl(I{z)drZ(&)bMh2o$KZ~!`aPS$KK7;QD>f)le3Ov`M;lQ?d|F0<>;w5Ans0= zziHo&Bo9@XeDr2sd0MBoift5YI=juCYosV&O+`+l=7x1I4M){gR8Xul*WT9I)^qOr zIG=Xi|JFwN`{yY%y70GFTTxR{VWy|8gSD%hr^|l<@BaK51OcNAX7%d%H$Z8>y1Bwe z)mxJPc?U&p#kw<{-Rx|gt>=0OP;-UQtU?QIbQR zkFOmB=VMXBxibg;&9E8yxj>=%6F#GxqK;yt`L@nZ4z^xSZu9{~jtlAs2Q;}y!oz-l z!`9U4`&r>!1|QZ%v6Es0B8)t$jl%>-m8BPWdhp*MxdVxZ6!PEkLA?}vC^jR4T<5u1 zJ1($wad&p)mtAh0@o+e@fFQ*!eg6i&%e_5SVdf`3u#;j3#d=&5Q6x9`L%`Lxx{%fDb6ZDLs{9T7zJD?3`D)Hr#Hgxm)e=9ru zzX{8i|G%8FR@X7-u-P^Lp??qKea@_73cii_96iN8imK&}>@v^U*2~esdhR?s7pJ*% z|DR{=<>VrMaAsRasvGPw{GsNA~pNPl!dW&lvqrY(r zzq%e&Sgpaw^;Ybu*qn@WcAGiV$#o_qme<1B+qC4YN$UEiM8@%hf|A z6xHe3ek?^mamwV#1E4UC;Hzb%Ffs9R01DlGe1tfK9PTmoytKlAV&B%u5{eo$#qI9` zitLm6wm@-vA73paMf)!1`+(xn5k5kk!VoDkhdM3;iXY3JBosAiiYB`S6i&bL-GE~G z8oruM@m9FgZk4;-1t`=u@e$$_en@d4rKTcK%yIOUP}HI+_V@`X_IEm402F~Pd^MYb z@`W3}ho1zBE%W#Yaf)c92pbx(04TbD2$4|KrYUwF5Ku%MUTX#vp~ZYPn}S*km0<5i zKvC}xA0bYWfD{4UI?e?Ot-CuV6m@6{&lLiSm#+@E07d#!zM4&uE9|J)+C|5KV&xk? zLYyKKDZe$|Dc(G4RUasx-{B*~DdYyj z&eNzc`6E#5Rke46urY%*93~`f9k7q9#xN~n0H7(5&vR_ z15m72Xz*V(n}S-45UU!)f#OVKK0=&g98&E6{B;dboI0v5Iga`?#gAJ8iq@0kqk$so zJYUVGpswx-w)-{!#nx0lLY%?|DTdWq^#dr9w0cM=8qgF$8w3=GFTJh_6h%6GHJgH3 zi>_m8WCMkc9v>l2;f@s3Z9{$o#nID4B^1gu#kvy$il;8eTLDF8JYUVGp!RN~pe_@D zV)|`9LY%@6DF&8o&<6_3%abJ(4QUF;?E(s`4bOsrV){+KnoW@--1TldM%Dld>l8jh zoFWJ*&it@;0SdbkCkcfLO`#;DFll=9Fi^zE&Hk^NO+l^2z%3)20!6VrA0bW=iWH0U z7rX-s?{r_uaWtYS4r~*w#k7$%E&|2lw|q65;*D@EYK$4H0~Gx}@)6<`;Ycy)h-x5E zj7nK6p=eA~+_4f+e6{T{8YueY^VMt$>grx%T<{hs{7d)!@(H&C<)j+Rg~ zp((qp=e4|`54JhKw`3P}}V@RRDEKeIKR%Tt5P&A_{Qm+dr)Y?w04r}o< zpRZ<9P#u-|vni! zZBusxh5t*wnoU93;L$d#zrtEL*GLSR^9i%YBmLR zbsG*kQx7Qo>hTfc6g83JaLkjQKyj|cTFG%}&=j$b0*W7rJ*W>Brn>Kue zIE4~Ye13ZUK2X%`cU(fDNmFzaej@fNaf%*Ll#Sr4*%Z`_`u(fEF;F;L@Dbt^%1Gfq z_alA^bIRhXghGp^P!XE>08h8cKrvztU(KeV_KUqk)`kN`T^~L|oT4#Om@hq{3~Moe z!XpWVHcg>x zO|*bwN4$X@P^>@8SFmpUjXg^SlN#G;IDOw@Lh=Z4#!dhhS{wbm8KvS#}{@vYh zQqd)#ICP4yW>ZidW!tdiJ5Wry&PRw-s3V2<)KR^FqPbB$4bEvh(iG!H2-f0<%e)Cd zQF}CB&8El}-UGQ-G#u|oHJQjqh*M}H#ngc_Re?f2TU|oYiKdt_TtJaCWBFd7`23l# zW>ZjWaa^luD6BVfH*}5q&V_4&J`#$R`ryGbfzgznF=Uu1}&J@noU7n z7gj2!3P7=8Djy+EF%BuNUvjMm6b-ArlTh@gDYgsOVt|hu-ZL*#;z`m>d~VR>$`1)eADZIn5W!kx?6EF}j+&OrSIbDT@np{&ptzpNM~G8Q zL5j<_G~8e<665Q(wEUfZ~SNYh$4Jev_|eQ&1f>-#y3%C_bd}5#kinkz!tr z3D1Bc&$zXOLXW0cr756TvLt2;P-vU+)iP2HNxO*8fn2uZBg83ekfP?24@ZHb*Ya)> ziheZ3I8OmZtp?5Z1BJ<2zM4%z?H8VH>zTn?tPSTQ#3>w*qQlGvYk}f?S0f2Uf0|;_ zU;)K9$9H%qC!`-=&8El~{+0AJP5TZ~81WI}6tj?GSncJdK;f=wCZQNWQ&=n$P;~Vf zngDAN(1ovNQ&9H?`|ogH3KWide1tfKGg3I8jzSeTBV>rINPfq$_E zbmgnr6x7vy`t^Gkpx7{gj}WJDM+&Wz5izh9N;MWrCh?s0oGz#UA|gI ziqE%7UIE3J#(ac0g%?toH6QN)6ju(dmQW0$DLjSGfrPC)>jD&-r}%0aDVCV@LZ^Lr zk&h6kSb!8W8h6BJ{$F>9l291X6rJ@1YmvP3YA8^Q@5xuQDX6{MdYU~x+nk}#M~G7_ zLW=K4)$mEwH@}ZcC=6+enr#FWdgHa`07b{z$NsBkQ&3lT_3cq@p`(t}=Oe@^{E#AL z-=_}HQJ*}nN{)l^t+d@e1QgT!Gt`0NkS|}&rl79wH!}`f0EKD*A0bY$94V9r^+q#q zZ23q+VMMP*(r>|xiae;N0TgE(_-Zx<)lr`Vr(On%TxUK)oFV`zPM4WYfwd@Dog<+z zrYVYrcMqoCkH;q{k~Z?yYzk`cK0c{wd!U#T$w!D&tVD{`$i`7XF~GA_LNSDZ zBJfZE?s|C-d?gg4X$ti`!CIu&Zjav)YxZidrG57KB%rvR#Yc!!Y(|Q)1|#t4 z$FTEjB@`w!#jF;BwK&^(dtF!y&ue@&n}V8AN7Xmrvj^%A_y}=|aHLRdd>5ZHuUl`2 zgklU$5nfL~F{@yuHBf{!=d0Ni)P7MC<+26VBEKCUAx^OsDV8^ThTjo0w>=@DFr_I5 zd=_+6*sEFZf#QV|U(KeVuI_IZ5j%iF-jk0Ir-(#~Jq?}k@9yzCk|Y#kX^KB>1#7Y7 z#CrTI>DvLmnoU71Ro(zu^lP8H~bO@lXj@xV+qALn!^8sU@b1InyiJj zI6jB3W>Ziz$}0VpHBkJS$47`$L?cD#=bs+|Md#=5BoyOmin?zE6z0pU{DC4Xo3Cb5 zP-~%dW)@D`6QB7Aaf)3?aeDA)yt-H3ER|49pegis2`JL8qznRzUeEYyHU)KEoV2Mu z71l!O9Umc1u?Hzm`DWm!FfFbrX>m?Fk*07-6;RwZT(TZ0YNYbjYzk^d**w~j2NX-6 z^AX|{`;g+G-w=F;^Ik?9356L=5h;8_$Aup{Ux1?jJHDDtL9Ined(X*0QU5C+Ax;s4 z6btLGz`JVQANG(?Orj|a-wM`Z@qW2qKryX$&;P2~6x4NbLhtnfpx82ij}WIgh!iS% zdH7wJ(V@d6$6-!WSmg*PB7Y5d0~F5Td^MYbGLSq^J9(f8+|5UbQ^X?0rIF(dVMYbI zO_xwirYX!W3Mfpnr>z8vIB&jMMv9Kdzgz)|Mt*#RIK>g9*rIKYPX+IO>ms3;LQ|xr z2`G9RzgGf^Mql`9HU)KEyh@hGPwlt-;UmN;jv+;%yMG?6#q(C{BotF=ibbUYilu73 zn*+t`4t%wY6#natAHZ59_U0qRDNZ28fz9^afMTcL6$ym}P0?CQU?66zmbwDPfDpb~ zMvBH>6M}%^!WKS4oZ=KxXxe0F14W&ag%XNsG)3)Yg0uE z?7X_W5GdvuSW76ZXo`=|1#6L!nsx*z%t!InYzk^Ewl>n31r(Pi@e$$_7m=dKDG?{F z_X`gRg*8nP86=?aIeiJA$?1{JSFI?a5HU-sDEnoTKGdazc^AX|{2}todchzQ~u-qLhp|GVXvUdt7 zR!qA;5h#Wq;j7sc)L%&!`t57MT8uo$M~G7-B88U4n@phCY#1-0u%jt*`~?)z#?HAw zaepLV&8DFC?vibR&46Nx86P1|k%Sc8b_C+v8_srmFQKrfDGo;qCW<2$Uy+O%I13J02Ey3lEdkDk^UC{CyI)iP2PTl(Yk zw5i#AggC`bq}a2yco)25&*jaLL)@X~NOfoHh>0~bB?Mr*ZE+EMFtJhKi zq@laCoDK3z2vX~;;VA%_<}1w*2k{vK-xRx8FX1+T*!~&zfCEWqye?-@ts@9>f9O1Y z02x|uIA6{N`6&dkj{IB=KuVfRGeklBE+9zerr7cDw#@v&qk4}mUy+UVD|!#;;#nI8 z!Wns}q&yIY&iT>OAdwe=28}+A=jFs8=Qo4?b{Y4IR+EX+kjmZ!s;4YCqy3Iz4!V7o3(EoeZZ=T&fK_TB;no-5qfhxa0+N;~&tgxaKYlY+` z(k%LYpe=^U4}~^*+UlGy^y7gQ()tQs2)fR`H3p@RZP;zt-vjaNeQvM9xL|3JtZxLR z4liNoj24D}Q#&;K7N*c~qcpYhSAx!oJAwmI*WTCK_;23v6=G&xsGQAVjCL{>o z4ZPy2@pqUD=XXgXMdyM|7S4sErKWx14YA)2NK>lzR?ynh6Ys;U{N$l9g&Q#`j8`ZX z)!d8SIiRfmDClr^UuijeR{j>w%KBI4p@Uz#C$PUK4QocK2FIu1ZCCC<;|K#~~m zyjd_|8-jRf-`NcyO82DYY!GVq8a_VjDlD$+b7_Vsh+!+dHp=qbX8=g}ob;~a%7=a& z<0U1VjY1LR%n9GU0CIG(v|L7zpC4Z@2ap}>r5U0ieghDsS(}#10CKOa*$xgQmho1J zhAsRN#P*8e2>>~()SNG8gH)4~!?)-94N5Tvke2Nm{4XtA7*9u#TnDN5wnT$ zYKYaHZLvEe947cehZhD(%h|K?kI*NoZ#{Df&dJ^N(hSja5*>`^wb~>E(ck>58-UE8AT5^>#4Kjj5&#)HO`0JJl6eS0x>w)n2_T)4I(cv)g^cg~ zPHSn1AZ>q+{sbVs9!tyFAe5?poN=ZGfP`dAGeklBE+9zzjjzW8$jsrFZg3#kj4x?^ z9NiZ|mS0Pb0uZ&y(sCI=nteRJ3qZUbr5U0i>UR+2e7Nfd02vvrsW`E`wVpD*Rkvq? z9)i4Wy#!Ze+d*kL8>CElu_s$h)B}*sC#4ypAPLVAq{uyD9Bf1*dJnPSKu+mZBt~oJ zox^!NZ~OUD5TlQSq(LHLbaz8Io)@Rn`yPTAX%3TyR3=7E-dw~((mNNl0K}-sM4DES z7~NJ_ipMqTs>U`DBe_Y^q<<2l5hI6v1~IBJU79E^M%!|tknw@(*E?`Ye6W+|aGgYn zk1+07nj(nN+(>Ck)x@aJ`oY%nek1_ zQr#|i?S0%E&|tfRD?Ju9i3(!XYgjDU0Uqq{Ui^qkn_<2l*0u7x9jybX6x=0Fk{ zUyb{;M+-sZ@>|vbki)yA5a5CswbfSA3FZXj$#pXRNc#ev)& zTag% zL`%~u5~GPm?s#08vn=yLjIP8=lm1DJ!X^!j0WnHDD@_y^qgfis$ml)#EPeuIc3qmo zbrK~;XWRBhx{og`rh*tXcp%NFq8LSKXgYuxDW*%aXfY}%EW??tRpxdX`f){`w7vo{ z8knqzS7^T9Jyj5+xnHG0vWgLPcqTR zNsIy-JD?ctTp7O#=0fj|(n!&{(4h^UJ}s;7yFiR)jFF~PO^jZQaZ-a>`8D)g2sdH^ z7?1oPGj538xhO%dIn2sAQPOf5XQfj{Mg*LbP6wnJqUS_e6VHizVlf)tTZf^=97qY{ zN%LzTY9q*@FMDts)^?Savq2~^3T(6-x8dFMr5U0inVk{DV{qRfSdj+{HjFhd-@OVL zj|Lyu_!~Ry^|u;}U`0axrR8i8>hlhTVBb_&kzc{m3{j9{JrU$_V^v)Mx!3pc1`Z^X z@hs=BT4e}QCLjG6Kn%x7%h@2*Hr!>vC4T_PbChO?g7^(Uko6wl!vSP>{z!$%tgkcb$~2wR4;L$ymoPY|PT8Pbr-#3(1X8Xl7A)(OQRMjLXa zX%&gl=^p|y3O-jd2E-_=P@42lV)QY}1s~0c{vk~i7o&G-?@){yH@V>no#a$~1HXP; zCsAUguGkLG`kRBB9)cM4t0&EU5c=^{TWNg- zVia8<82-n`l-vR_YSc*@B&!%vhu87+yi*`XlX^%~D;J}#XU3oy^^N+p3&cokpfpb= zG3xL%1iQY*EQ4n-7wkqzBSq(eo+++_{p+uXK#UwGNK>jNM)msW*}|+e?{xMQH)1Y2 z(X-Mk?*sBo2ps+dI($Y?X}OHEGBNYqO*ki71Em?F=cI4~o|CK>Ym@+Fa(M6hQ_6R* zV=gqv1vdf6i$7l40Fu2+TFwTc{;YqTv@Z)lvW`eIL_rcP5ag}8p(23nX};H-1KHz2 zgJf1O!WEh2HD)V-Jkgbw%Lt;@`|||=S=V2hAqo;|hahp98x#P*=1Hf-=Q_zr+vA1BQa1+kfpAXBE!wg-@V%{G^DAi)c1kY}UQ z5#&<-{n zA;Rut!?X2BB2-ZCbnvfT$gjW{85w`5;KjO|_P=5w-jFXblH)esM)&^u9(D zL`t64;4$1Wiv2AO5)q?;D`w#qJ=RSdKT3O4^D%!km5I@ss{&Wk$@%ma#OP^VX<9{M z)PJ=p9@mTFM(se19yXFD{gW8Aow(Q)?ikHil_rXd(MpT2C`KQx9^w<7M>V85TqjXt zl;GR~>8`9_XANR>p`$dTiefbNRJ}ysN%&t{=^WZV8NW&ulMA0kAQB{~>c;#U8d2`s%e$7oI~@3Ev% z2Fr+LFOkJH-5p=Rt>2xfXR}bZU52_I`hqM2i4akipipG7T76m>Sk{=;UBt1(ZDOzt z?sykjRPWS%1U=U02vN^wp_K2L)A|fx3BEvth_bkELKdewwbDQ*9OgT1<5(X4pGC4a zdD=R;>e|}d&-1jkUudK#Urj|$qvnt|t&r;3!sT0GL>Db5l0;-_^;}2Xwg;UbHs9dy zNE`1s-&UdTS|X(~Te{k*J|0vJ^T69M@jHeQaTV#3a-jttney_W+OVsA*g-`7lP_gk z<;sH>3^+(ciVM@U<9|??a%SZXhmIO`iU{I5iZUjH{Y??DuI_;W1OJ|Sn~|Rj6q?2p zAyrhS3+KIzy8jJ}^7qeE_;ig3qn+u@(PVOBqj)3W@ zvG36vZsNp-Gp48E>#K<5dU(M;==EDiiF)?*q|7O+z}N)Ni+UUpB6?n?gkx*T`OLzP zlDDn*wwYeOt3^jLSRy{$L6!&o4*Ud`peUkVMwYA1KiUDyfI~!xD2rhvvec-%))m%e z>F&TVj%8CcgJp(JGO|n^omLxInja(TWn{Vjr1u73Nxwvdh_ZBuMi!kF_HL67wxa&059I9L$%Y!>S7lFEc}V}Pa1jtCKDQQm_r&-7~z1eUe(XG|^2 zd+gVK221Ww0ZVp=j1IsuyAe^(W}%Gso8pW5z|upF2oYr|*^ex5%%ac2P%k@IaF}Ds zJXE1=)N*T$-4@WXVI0^-i(5pJh;3|7^TjPMYj@T&unqH6BBe6h7^S3w2es;?W0{-gjpaQnx+4O_76{=_(*F*1^gKaF+B*JLhIQ7LB8*_`-^gL+H0v#ft z0^5*ZABag)%w|V{Z5-}RB*|(U)B)zqt=$T2W3vGfTe)p)JG=pHqt3eySHU)dM-hQC z*@k?bI=H5}r~7NblyICxWQtCSghO}=YyV970=99}nuw{IZTN5b`3vGU?Axc2X8lrS>x-M3g1`EV6tZ zmGA^ux?8SmXkFf82^SbFu{%y6i)H9;{E4w4vx#~(3uPP8r%tp6mgEQ`M3lwv0Y~;hB1y!YE*QGvs5jf7^&QOVyC;!SnK>-L|jGYhy7hQ8Yl=H$4Fh!i&`y_q!; zQz^F<-jN+0N(6BoMVZqRIlw5E?tB*Za13<5oOW4fh=Pe z*X{`{a~?fi#<7InVXz!F6YS8w8EcvX%akmlUPcy`M+55uOWH>wM3hDS4zk2JJ>LN= z?Y3t=>?2&$|8RsSw6Nfas!q> z`QECwr%UjsAsc$6@K!#bjj*Fz!LJC2oYr|OhuM^E>WL> zC1PgFFpj0*F@wdybSJWmygExCShReIdNvDX=Q-Vr(}Bfx4G|*B^7JvX#E##%6*ef{ z2~$%zmTk`}G^Z$|u1K}_{;KwH&#BaoND?t8{lv*Q>Qw^XMS(eax)LdsnbWu5jq#wm zg?)bj=Jd>$h^xq)X05TqBRgs7VFBjkx{8SUCv#dk>})ca)7DKyq_{aXP7#>Xa0Gro`Q97ShXGbV~GjhNsX#Z)XUYQ(9djrfTN2BkAy1n4TTJ zI=%Em`NbA|tliCT)4-Hrc=Osa#2oXIm3D5Doxc}&# z4J?O?zm2mi-__DG7%YQR1uQ3LhHnEFN4X+tJ)4Ev3C!yr^8%I}c_KuVB_;z|wnTVu z2bKe)rp9tCH?tTl&XId?U8XE+r3ftZCK2^)7HVCt<}Ex2Ed8vB5K)%UEM&QPt=0=* zu~G>sttWvW5@CtES?PnEXoIaRD*Sy`;@3>vwRbp)0?{{O@U=kCJ`daVv~<7(<~Ml0Ly|U zb+>aYdpl0pv0Lzw8B1DvB#7AWL5n*Zw8&rf} z?Kd1tcwvR+v^YiwsTR(N(*<)1SxqE~n3L|PsW|GdT6Wb1bGo~pNU6-6%F>$SLHW$@ zdKJv6Mg$R8kvXkcHx`de8nk0={=Vvg?BiZ)5v%tj5ep92V=1@r+rS=gvRu_ zMg&w~POtl&!=#|=efNMl<=-KaWHl%108M+<*8_9fmrBG|Zcg3DT}E>nvq=9fnA7j) zM4(LO6#i-=uBp5CvBfYYPG=FBqEn(>bL*;$Uo zuav>Et?3SA+1uMf6L7&#S<4Qk$q}vrzW}lLz`-1(qOPB1DuW_cyW>H)whSSiC#$x8+!J zmsO`f-A*;whb(a&Bea0!lO9peW}&W^5YO%kz*01v2oYzI8xAk_uXBG}9bnnxrhb`Y zNngQW>7OPrCoR`Cn}9{pkEmy}d=~cD?IRQ2fhA@g5hBWxUL9GIg2GS=z?|OnC6YwUDdqA?9QC7A#;*f&>SI8pRAx?TTLqui>BO8F z3Fb6m7!g;IIho!RJXM|erD!ymQ=};o^-t#X=V_)km{aN`B2wI(0{7R!bMHLu0RFIV zcS|CO>nO^cwEGA?tvk2)!#pr2dj}$qV7QM2;#=v59{6nXi zG8C`u?X_u5fo0OTO>a4tO;HS% zFL~RMrNC}$60jU}Bi2 z$p*#zKH7pG-H__`t$laFoC=x|Nh0Qyq~L&~UhzZk6JSotZHSc0%qehxGd!q0n*Qs+ zoJMIAaTS@<9S^~~qRr2*&H{7lr9(vhlR5P;{j37!q|k?m6gQ{jrGk4-2J#E=rx`CA z5J6l=QRdWiqTmVf(_wlZU`{HdiI6Iqld7U!DwxxTDMT1;POqN`9#xRLov8widC-vv zsKA_h*a<#TSv>Ql3z$=?2azPJIZ+3=JKCr|n3LNwBDQjKa(kAJ=JYJ^v;&ya==DUP zOy(pXxdbmKxm7>V5cHynOwlRvKrkihx19a}%qi{&5mPmDTGpv$H=Lf0!bfovCuk>Q zdS09?*a^PP)!h%%v%fk~FXQwy{j4w*rswvKM2P5l2~x)C**T?yC9pIak$aP4`L&0^ zVr?(5jnB6yya$%R@kBkFh1%6Ru2+o&mc|xDh$zb<6=dl;xd@*SU!^eY564m%!(eHh z7>1*^+wQygdy}4xiF!5*wX3buc)uK2wy6;zqAc!>ktL*T44PAapSM;{<-1yTEQ965 z;H}7VuWkK4nb7UFY{%8`gxIX_;!m%WuVz4ai zu?1O9%u7#!b?NhtsAscK>vF$u$`xQy{6>U`vY50)mVph;@yEv;SI=rWyS&HJ{?Af@ zIlVX|FsFlU2jUyy`)?wWM9k^f5y4N+tsFXGAehtoZA40C=4855aL;LXzlp50nPRp~2K$*CrWg<ZB0$Y!>RCQ<&c_4`7LUMudp6^lFVP`6i7g!>+cp+X(GB#71PAi#_4IT+$;#M9)i2Z9FfvUl-%kyQg~&3*uNx z?lV|?^#$(J;ze; zkip{eW(}@ONrWwa?`eAkQO{Hw*-M0ovJ`YgmM3R=%?FmkK0SxHaJ$+Q z2FuXA0Az8}TR#_itaK<*&t{?4Woz$gV}YfnIT0etlGzzqevdv94J@0QB(CIG(q1rF z3{3?0HS1nzuoqa8I}!D47V3K0mjAH>ur%pUgov`Fbw!pR8L#&M%UP?aKOD==R}2>O ztDA6LOhz1A0xXB;5cO;p>OM`Imy6ATW%L3fM3m*GF0%A)7~KapsQC2#`mW_YR`R++ zb5h(s5UFafR6Pdfl#ov(iI~&=c7h*Qo*FcB4VY7#A|j_Oz-0`W)2}~7 zTt()TdTb7ELa93sW`a3c*Nu^$sedx3TO-a-0dvZ2LPUz2Q@a;}*KnV6`QQy5)nA9{a%GwhlRWzrfaMdqhPQSYoVYE4=`wJe7EcN(p1&ujYp9rYH zoGfby9(qu=nbHEx>9rA&B&#`52RLQ<_lICkR-=g6%FU_W*_!xEZZG;39)EVB}%&Cl<@6tb^y$2mkkk9HFFw#%f=d}XXs3eiQL4w z_=Yh(yIcyyOZr%i-zTBhU-}aD?CDAEYO~KYbb<3SE|>@rJuer#<9WGU=sE#dbQ5AO zaxBN*F<9(d3T$IRVawORl5n4>myu=f@Qs~;Wzq{GM3m)NPh^R-iNJ4+Hw^CI!mWHK z*z=yj(sRRlT$jB$N7n+2djwI>W}$ZI>~|e^154ySB1DvBPj6&N4lBI|>tg@#g9XPD z{)xfzXNcfF&CT`s^?)VgEm6;Ap)Bx7Rt^=uYuU5wP1Tfw^g+Dn9pvIO@-mgOb4+rzr#)KaSJUfyGV z-x(}A?*7+dVYAgF__bq z)0ntbDFG8ME#RFJ#$aI0_HTgD-kJfPW3#L5c6tv7koOiX)hv( z>nO^cdiE4N8>HXr>sc_T(F2H(Dw@-SeKj6}IdwE5!f11vEnf>8^IdL%Iy9!=NFtyD zb9z2T_+ZOC&0%0pZ^jZyvYHchfLguM?}ItXO(tS1H>bTTmC&4uf<7Mub9!V&1j=Mi z2i;@waGQ338v!^F@$9_l5;uCOQzTYH5M9+)YKs+y5H%IIQmd7C>FFBUn zG6u_y9lm(ornfnZ@0hQ)I#gQEW}!Z<3(mEA3@kRAh!9bhSq8|GUu?G?SWa|(*3P4R zS4&@BgZ|#rnmbF8<)YhE+@Uk{iF!5*bw9xBznJ_D@fb zdn|4pgT?HFU|j~iO?wV3#$H4{n}zzTDRb+~y})vI5fLKFqBk5_oIJj}!vIwL3jF!Pfl88BF=?WfZu)g;`0L&?AGm%o6IXScz+;d7@ zQ5Xs4qo*TZxF;P@VV zb6fj(B8clK%AC|p1m-m2Yy#sQb{E z?bC^X3e4%ndck9f^&WmI0CRGDLnO&+PSgRm(>bmN=5*)-5nH)Ad2MNg=5(=mS_3er zYNbS=Oy(4$d={H_@Gh+~m=anwhDtAQ(J8S|FeSdeE=>h<>Ry+KshT;(yx*b-J3%K; zqn+HuiQT}MoZ+-pLYLR(?nX&W}$53(?tjTH3>guB1Dv>@fc*eKe@{# zV9|Bm7|yW-ZD+7}En9-?Vp4M*vaDJ})U#Qrdrm=fUf|mzw1bHdQ5N~J$fBcT-UV39 zRQDHiEWdU#SdvBxu9xS_->X57J4!14I|`r>%7jb)ZZN@ccDe`iZP zsF0;I1+Ic*+!eXE_h^}f+mgy+X!(bqW;M?CizU62)1!;9uX;S8;{-zo(
(Tt_n#6+X&l6B*|(U)B$#T>6;C<;lGQBt=u;9om!%8luVP? z0Nc2AhzOL)Hr(^BV$&KMtZ~9AafZkgof3Hya7w&vzZLJ(yoe`as%9GxZdA*F={c;< zO*d}h6#jqWP`~nV%2)8|!%xqDmIMoAQJp`i7p%+Y^G7<* zFYmEa=NK%uGo6v8Y2aM6z-9M{dN#`^;XS9)mOf~KsaZsbD9fqo$kIu#qzP0 z2XngJl8CFwoL04(E7*ii*J}gjv_pf4`X_U8`W$x@%xP^$B2wI(`rNIH=l*(H3jV6& znQlZ7*HM%?`Ck$|8?;qRyA;eRwl5JNJsvt=yc-nromr^|(5yF__aK z3nEY^bNaS41)KJ0WTyczy=U4HnW9r-wiQl^eRAUs;XaM46A@E2bBgdj+YF|sNgvmh z+{B5w#F(CL69vx)1Z4Z*uY4365%uipN!dp4oFTz*UVe`yLPXC?jNrU1n4@+CSR&%K z{o+_6uP|6fobtizR^?GJzS-~6HKLx)LVa4NZqWySad-0rB1DuW(gxRf!=mE)z_K&y zX@BqXT`lw)gQX;XHnNO=*)bk=Xv_UXJ)4Cxr!xah2|7KQSfZgkGG}g!JMkO5=kQF)S-<#j(X4K-LHW;J(x$NRAx?{)LP-GUXmNS z0?g^1Eptd^V^of(WUiIc<8UaR*iP`pfkhD${lT1e93hfqH7Du-KV~M)19RGXnux93ob{}F=enVX>M1=I6{2~p3Uo|HL_S~3_v6>>BuLPXDt&1^g`sXsf7gPmZ>x^`t8 zOWIQgi(@sxa~s{7zQx~1OWH!zvstKJZRPw#BUod%Xd*a` zd{?`f&R{vCZik~bclSixp>+-t^=uYuSKFP|8oxie`2-On$}++gS?=c7$LE~3nEklG zv0Th#uryw2g)DNeZ%06nU9}_X*(}t3ng>x^@c!92cOpcTMc*A+ls!}KLyx`dU;2w< zIhM^}*|Sk#=Tp1C>_ zL|jGYv?;hM0pnnO!HM{>hxaDj2)~bGmz%h!i&`3kQKYZ8&~@K6F%C8WF^G z6lG3}yQv~zdi$n*;GWa<7eq)E&FSXGJc$jGpCwg7U18|j|Mfn&rO{0 z0><=gdD8`_r*^PO6X^BMqltR<^rXz`aZ1OTa9&QC6CtAKrTKiEo>Sa!;iq*aKTCS} zmhS|?g$$N;*6zr%En_tPGRE#&Kcw|+7HTI@HPE{bEF_WaUPL{ch3c_as0HE^-5~w3%6R1;>*9 zhrzPNaRIK&?=v16z|vzrQ7K-;IcnDw>l^yYzlwPQO|bVYE3}oE3brxQY6#Q_z@QHHd%; z%&FUJ!53}rYqU27b1LpYB*|({)B*Y_qtY7BYn}FG)@+2f;rvmO9aYf zP8a)r#x+fQ+3ONaiHinArr4D5>*R-1;%khZE0|Ni;Y3W;%;``4iPd0wwoP{U%}tyX z|C;o3PH8s;pQPC2YU8gc>pdgt+0&D{AMk0pHU18l@mnH9?7Yb33eHQKM_oN&QF+CBYh7%XgI?geXhaGF;;;3fppk z<mnmzB5K)$t0A!JmIi3SU{o%OH zH#wFyTPn1THZ2DrmAu-+Q(zmr=MqUGwqftF8OLM&v>#?*8?yt5l*(*Fev;s+YWL*L zZNWBbY$oC=vJLYli}A?zx+K2<+qia!i25hnaA**R?{^N0CnCjdW7bK*Q`H@YtnUR^ zN263Ci0dfIHUb~1A>gMWI8)J%yFxoaoEfu_$A!cGz{K=M? zid%$@>E>nYZ0#`5)7Hz$&DBUzzM6`hMoqPzBe2C)rg&I@ZH(0*l4P|F>Hv)`woL-t z*wl-Nt=u-k&l#X?OlbRJ8Q8|0Aw-}|wvo8%7p|#>N&8hWC7xLknPOAI@7fAHh3ii~ z>HxN(Fo%e#nr)Ocu8zO`KDPC)TinFi^#6%NJyjiLEV#@4S@mYrh&h^xmmhWmo+ZZe-0tELqsuw)Q-$%Q-kf>*~ zP&>4v(@Pa#@mxuSh_WQD#5Hc!vxOY&YR_-fn8~sH+Rk9f&Gf|i|Nfqp9IQ)&G@_o( zLhWiLEA#N)M(dYEh$u^ZAhLL^%6b4S(>+_Ib1a3s7%cH)<{-l3_#nA559bh?-;M&-)yvMTlF<7#krz1;5?OuC;CB2xaXR}bx zpZzKM(F#~JmE5EUAEIF_de87zYi3O;$9lide<%zGwL&t{=4 z(8Qn?z6tZYClMmb61y5%44U8e1as=>-B$wIWHu{@bjrY-9<(N6D>tXpF0uQE~3 zo}ScO*J`TtSA(4(x&;v;dS3Pk&P#38*6P3#wNp)RS^2J(aDu^d%yvAoq||O;4=l%G ziFz4X&gK8aPwOt6CPGA6qJwdbLtXUo3GvsK+fH&Ur_L}~f}RPU4T?FcfN#U|o=w!V zStxT#>C-#`)yX7R_0Sq%Iim7@)bjEki;QQm{JcLES#0#W;IDja)*$NH zER>yVH?qA7EWWx#h$zdt5M*ic%ccle#`T|kkYfo=WU#by8Gz2tp*j6b8IDv}`qg<0=G6KnktAYH zTI&|!_HCb-cLdC7bT*MvnK{`#7JR~WD|7e^FsFV6L|jGYr2fhmk1TZ1HbXEcy>CR+ zKbh0;$V;`soNRv*k>cjG{(};p`+H#*9igL^S3e?MMXsYLbMo3DFsHlyC*e2wH!Bb! zRWzrf$PtskoX#r~VYE5DZzOn)$)7)6RzhPQZ$<=E@Cnzlvx2X}`oD@D0p?`fnn;q> zoTvl*;%FWZ=H#VC#8z%jHD-@Nb1Dutz7OVfuqzQLlR1t4qKM}7De)oRTQTcRWQt7* zKh^a(C8qXRxE0K4)<7brYUUKvCPD?K=OOLZ^8V#JLBMs!^z4}6guR|tGhjXRddnU} zy^Pay_JYu6Fg-gCB0|K@i`;U-c^OuazYJIcI=)@Tv6S3muby^JjC z6aIJsOV8Ovh$zeajmYx+rb+`?m$o6^%>&ANEb{?_WrBym&P^AaUIUhUF+{zLEV==; ze*sI$6(U5G#eEa9M0?I&2`taTti3swv_}k<2X~EdT}HiHn*l5*qKSGo3w6)QY~cfZ zxBGyDM2INM>@Z|$(6W{*uqef0U$+7ITGsY4`*nA3~Vf=}y0 zc1^1Z=9I5bq*P{3O=qj&LFqc2e+%YxX9N*fkvUy47rbLYD{rADnA2A?BI=*aX-oHn zWnfNqZHY*6b28r{csA(ez!%e@qkNo+Ag-e*bMk4^4%Kj(wpnes=d^eM5mH5SS}JF| z1I$S;fC!__DfPMF^)e4sKSe@gZd*qLRA5dsYuRFpCmmdz3g%S2g-DXsoTvjFoO{p+ z%&F;KBDQjK%5t5A=G1jg=6f)wk4K3>nartZXhSrotJkv~VM;W*KxB$d3BN?aln5TN zQWwlgBaw)ynmJXAY=FP_G&R#$e?|GOcJU=+dg|;Kd}Dmf8-M)9cx;U!(t7svr2g)- zyLRsZoR^JiM2OgVk+a%@t>u~;VF5cqnElVi9LurS43^B>g6ApIrVYIW*KLUlQ7L&Y!>SNS)N&B z17O+Nod^+SF^ohOgT}e-V1vrsZmtwq-eW8N&m!5NnD43{sUqmMwu^U0fjPa}Pb7($ z)6^}3XO|ZY-ER-(^yvtZQkgjwt`NMd`b*L(4KSy=XNb6p%*kfbY7~o@YES-vIeA|q zqW;O8-UW6X2j=uHiHH<8r>hSI=9JN@i8gdp#vLMv>nO^crhd{uKwZV#-N2kuQi+f% znp28F_$V-^htG&G+MN7$30^c->!ET#Xv~sKBA^0u>T4wU8hWYspn70V3*Qk*vYHch zfG2M2Mu9oa`9#E4ZcckUPe*fVa(=*7Fej^GB2XrCvQ}?~<}_$|u^UW@_rHluu_@ta zB$yJZzl#=vIen}-Svn=EW==mE*TBC!yBau|a1-a-H^%hT_%3+YTB1{a0Q9<{2~p3U zp43iIdt>+4a9(t#5+P#eMXrAoPS2csd3c}Z@#~-?97}EqgJr_n(YUKUxuIhQEM;Gb zdNxam@NIaznm@(26b`SoN_r5YEWNfPi++>MwP9ED+R~$bQ29=f{)@pfbG+a$=Y>OG z<0o8KZV>fs7V6WwvXlk*oYTO^M2IMh&JJYhGS2E4uozcMH{n=P{A$t9(#8EY$9380 z)@~Q9ORxGwJ)4EH^8wdhSVNDQsS+WgEFGeeXs#Ydu^%|fk9*2iQGU@F4s>n8OlnOpi zo&43QH`s<@V+ ztw}4ejrV3mlB~8t9iZ~8PItjJ{!AlcE4PglIeWB?8i`psU>ia9M4(Kzp*y%0+Q#A3 z7Oi1Q*v=s`#ioQ`ykEC1lw5dNyJpmHoo^dhQEwq6;RZgn>ca*pE%T0)m!ED zu-83@_uB})UUMx`&z_!?Z5&E5#y3$;3nN0r&Wl_t!FjoS>OFp1rxDY@k7J1qX0S}y zF8CX|Z%wUdyKT6%|g9LIBmM$Fkm^}j0h2BQQm_rKErnG0+t)mKijM>-_an!05p#hh;sOyO%A&XzS?+B6UIr{R->nPhSc1YC zEQdV>wqdzF^$DxfTN(RLq2Zx@*dNEMP~4=2Un> zaGxgjYr_CAr&1dtNyMBCbOh#9BWhqBFem*vL`r4mq~BfeY|xRiRUu$bF7t@Eip(kD zdoYSclelIlz??2DCZhhyobvCf8=^S{5Ru~Ml;NR>9rfwMwmZ;KY9T}r*HM%?WxBLS zz%~iphl4qpMi3!YG^fU2L$$!1rtTxcXmh&PQ}8l7r##OHXiS@9L_h`RG~@n4yj+(o zlrIKznthQ-lGU831N`cC6z|hqxlY7ZZca)Wv(cPh#c5cAIrUB>0%bC%pigbloZMSS z;IFYJW)PX8QzCIcod#G_!v9Yk%AB$%3!X&Q zdZvo^X#zSD^)gOR4U2jsVS4&@CqhKeONrpT6fYi$e?u4P?%vL^WbbCMOn2{r*X^Z* z;qq|ZIt(Q0Wn@Xu3^D+grz43FQI;<;xW=wAW~#6*y9z>bIF_gT87wMw1oPjc{oq-^ z;`x)Pmyu=csm}Sp;!r14dJv*41qYBN)_B4JUn1~i3REjK>NDHEp3Z*2ZBBjun#uC{gC6x91?f&BZI``a9 zpZo#u*W*6#IiL4=e(vYo^Eo5aoKiY7I|R$iVk*F#;vSMD33K}6QxGoSgUVl7fjOBx zCn@!rlZiHCPBZKhXM#C(sU~r<%*okuJ6@Tg^0T*KPM5!us2?(?kxlQu0&^->sr`QQ zrOnB(kTIvcs3j^eqq?;vLBfm@nbVkH#+=?fjGYbU)L)B)G|-&ZHeXW(=H%L)go(}R zjRJG5X~eH{b6{fr+nWT)Fef`T=0Fseq|8QOPT_+{lDy_5y1=ShH8ptVlxaa?>o=#^ z-7C?Y?l|S^fH^I-A%Sw4lVb;6G$+d*`PjWDPA8d?UBV&;yTsO0JC49Jr;s@$reWq3 z?{j-RbkE97>I;O96P3j2o(s;J;;Oc-wX+w@^|*9W&+ne1=K-7C8{_xRK-JdYFGBLZ z6f*b4CCvSIVCi$#=cK@L@H~fQOiSiXltxNLxNoC0gVgg`M87_)$|-V#m0(*g36W&E zaT3S)(3_Vgz_NMy^&UIxRs!EN4okvw<_x5}yR8FYTx`~ndOnM2TuN*8LV!ilorFlT zWSv5m72gsj083@ay*UDl$0ZI+OLOMAX7l}byn#je1gYno>5wuTKQ9NmFIEB;k5;6f&muaZ^see^M;MpQS|mi0B_$47 z(u~qS0L#3;BQZ zGpG6eJK#n2m@_gM%xP&m5+}=?>;~;cb4qWwJQU1nrY4E{A#*xdV|5zLDMXJ%N}E&e zYZb)2;_v(vW|WdK2@+L3G6&U+8D>8T%<1AJk|eJ=i7xPhO6u=mPP6SvZ2jhx-)bG2 zlUcJGZ!jl^`6N&-bF!}Nh33>RaBv^!cu!Z5OsOv6*84PeiN9Ws)B|(+u$II$%$!Ou zkHWKeT?3E13LWRc4Nmu5SJ5Bmdau!b_@83Yheg8nF9e#Zau;>gWA(AZNXOQJ< ze8nAL*|dA_E`cTG9}dg)EHfMzgFbIo!?-*PCG~t3kvY8^P<#(qJ{%_@k}M&K$fDd% z;{mWd=o4Ncu*CkyVaZePi!5pC0k*(0eh8_TlVxwodUas&vm_ysEI~=gvaO5fVql3e zZDHhBH^;&saaao9Gyh&+blhnWu;h*)^?Vl5XFue{%?e=oVnsqES^SfcB`~*G4;H9l zQ}_KLu$(ECX-+ncn5VV#XB;R8b9%gxBuSW4;2M8iz9+eULUVF*CMorqQ@>#5eY80z z^9sS7BG!{QS?08IwI^QL+!wb)z?^*CNz@OS)5d1zRbWojw~`PbB7p4fjLDakl6ao$+E@`&FQ0(%Q7&h^(iD! zE_2E%>5JwxO3wj*ty9k=nNnTC?H{H~n5?Nu19P&>B{2;%r(#Q;dC)!E7$hVJ9Ve)a z(>-S?GjAqeHrB@P(1Qn(dVcp5eeX0MdA$R4&q1R|h}3;i@Hva!bCFxWjli-@RrjO7 z;`4&TqNmBM1RhJOwBU1F`ZKBLvxrvcFF{4kfF-3136W&+I)^MT@|Fz;7Vr75r~22e zYHqJNEKQ=j;R@Z|sl6(&Y*N-fES><(9SM=TFA7VU`!Z?A&2PXm zJu@;xV7d8)!&0Tmtk6;AnKi%?SU~FeETUEI%Jldg7~`=eBt(*BQ5uf%O2eIo!1B!{ z=9hrFl^}CdQ}MW1Ki0z)dT?#xRbc7nPU`tAqH+1%l#E}nc2%qiM~q||3l&kdMg zxNJkebq8}gvz^4rGN<@v`|!#(6jgizbD9`HqJGGnw9n1M{}i)1Kq95h$^AFx7q0eI zJ8^er&%-20m{B5g8n?;>0k?Qf%>Z*+5luoGXihd(=P!UcO^zjDVskR}V-B|uESpgT z6Z6j`5+K8z0**4r;=OEd^BBzOXevpP*PKKbSn((VXJU_M8KA zQqCoTa+%Y}h!JQ`5BAmWf-cdbkYq}A3Aa9(*d;caRxJT@GPp-#8fH%24VU0kZnNp? zyM>Mu;K%8n3TezraJuYJFm%t2i%31cdy1ZIcoi(`2=_(bnS@B)7X>Tkz9bw8(|}b? z;dpGdz~UXqVX?~VgR9zJpD7=KOnpv%~Xg>|<=U>Vzv)bm+HD|EqDKiuu!M4N<2vJAa~EaC4;ZGj~@ zt9*~Z^5!6iMPmc=tI)8#k`iFK`VXn+vxw)I=@WMtmoeoeM3QCTRb=UPIr}=W)GQy| zYH!`RJU+x>@$1Jt3v3_ikEaxdY#{Y~7LjfA4eM$EEK9eO5J?u3EMys|Zn^YVgcyYaoe~Hm42ttO?=RtTt?b=0&`N!CV_I9 zQ>y`%Xim>{t#DWM$9$40*(Dw^T|(Vv>OwH5=EWqYVdnIHNHYWIo?)J4@j}PRjpB6A z@NvvaU~|-UEp*R0ex#n?Jw>b9mVs$4pnFCfBq5UbrAIb)Pq)H2T-Ao!_tpriTh-D} za9Dg8m))z`@gSTfQ{h$M@44zhUvaj6}QOR#0aEP*8&TMg+u<6Fi_Wlns{88Z*ui8DORmK) z$fD+}wHa7UD@i?{MKms>BDdiCXpv3+{eBUWEG=&!OMFHfdtkBu&Bsw-@lD~d+;`}S zEVKMwvVoO{+b8NID36W$`&O??UzfdPwp!PX*NEBGIQe~P`{8#2T z$aiHEwZNP@I*}v^bJDmKhRb(1oz|_uoF+Pxl={p`Ie^(4w7PSkCzzA-dJ-qgoYrsi z#w)9|EJJge;7+1`$ed2}_&5>FY2`K&DQ!;Ote6uQBTn5mh8Z=@j|2%bN@Pys@{AGi zMfGGQFsDI5B&31nRN2PC7|dyOCJlG5-=I5g%K!!O5R5J(H9%;Vn zFqo5a3`vsLoJ1FxU~{qx%&FuwiLKw9Ogj6bIdvcTBo55!*m)8tmpNI-Oh9vTm||KE zU1G{5k}26G9GNar`qjo3%*ia9#5ByDwv6}Cg6{cKR^ol3<9MWVy64(8s@OgCOpE)% zTpvgDX7BgQ%1h5ORz9tn}WFN!y@dltAh#pg6`JsaunuUpk>E^%0N9y7alJ@OPQ zfMts*spqqZRyC(1uDGA36PRc-cYp%bvI z_9yjx7Ljc%w^Hu`<8t)?36W%ZpO52mWwx3%u!O|x77Hv7ayTqYmUqQD7UmE;7Fe|N zNj;xM^qi(mD}DSo)&q}8h$PFK0%X~G&94ctgoXTVa-eRG73OhRLXDUWD=rgv!~sji zF;Xul%ZRu@d>`#_A_0ZYt(E%pj5S+_VW`v&UZxMXz?jsX_C*QB1$ zBHDkJtZJ$TbF6hO36W%ZT!<`seQvjd6|U`U^=N@*b5T=$@d`I=1oQVT8h37fgX{QZ gBMFeajuAz89lJHY-h&1 | tee -a /logs/imagenet_vit_pytorch_09-13-2026-17-32-42.log +W0913 17:32:44.203000 9 site-packages/torch/distributed/run.py:803] +W0913 17:32:44.203000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0913 17:32:44.203000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0913 17:32:44.203000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-13 17:32:45.752320: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-13 17:32:45.752320: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-13 17:32:45.752320: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-13 17:32:45.752320: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789320765.775946 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789320765.775922 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789320765.775961 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789320765.775921 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789320765.783617 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789320765.783616 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789320765.783616 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789320765.783617 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789320765.807591 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789320765.807591 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789320765.807594 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789320765.807593 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789320765.807619 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789320765.807620 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789320765.807621 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789320765.807622 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789320765.807622 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789320765.807623 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789320765.807624 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789320765.807625 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789320765.807624 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789320765.807625 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789320765.807627 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789320765.807629 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789320771.504566 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789320771.596808 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789320771.689184 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789320771.791394 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W913 17:32:56.507268257 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0913 17:32:57.519462 139667382396096 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/imagenet_vit_pytorch. +I0913 17:32:57.519467 140507706811584 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/imagenet_vit_pytorch. +I0913 17:32:57.519462 140560219378880 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/imagenet_vit_pytorch. +I0913 17:32:57.519504 140384943371456 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/imagenet_vit_pytorch. +I0913 17:32:57.542615 140507706811584 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/imagenet_vit_pytorch/trial_1. +I0913 17:32:57.874481 140507706811584 submission_runner.py:242] Initializing dataset. +I0913 17:33:16.286662 140507706811584 submission_runner.py:251] Initializing model. +I0913 17:33:16.659709 140507706811584 submission_runner.py:290] Performing `torch.compile`. +I0913 17:33:17.722879 140507706811584 submission_runner.py:294] Initializing optimizer. +I0913 17:33:17.723803 140507706811584 submission_runner.py:299] Initializing metrics bundle. +I0913 17:33:17.723958 140507706811584 submission_runner.py:321] Initializing checkpoint and logger. +I0913 17:33:17.726554 140507706811584 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_1/imagenet_vit_pytorch/trial_1/meta_data_0.json. +I0913 17:33:17.981116 140507706811584 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_1/imagenet_vit_pytorch/trial_1/flags_0.json. +I0913 17:33:18.021770 140507706811584 submission_runner.py:359] Starting training loop. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +[rank1]:W0913 17:33:30.378000 39 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank0]:W0913 17:33:37.289000 38 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank3]:W0913 17:33:39.151000 41 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank2]:W0913 17:33:39.498000 40 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +I0913 17:34:53.777874 140488489289472 logging_writer.py:48] [0] global_step=0, grad_norm=0.366094, loss=6.90776 +I0913 17:34:53.821201 140507706811584 submission.py:307] 0) loss = 6.908, grad_norm = 0.366 +I0913 17:34:54.415928 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 17:36:17.812426 140507706811584 spec.py:346] Evaluating on the validation split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 17:37:26.016071 140507706811584 spec.py:363] Evaluating on the test split. +I0913 17:37:26.336865 140507706811584 dataset_info.py:707] Load dataset info from /data/imagenet/pytorch/imagenet_v2/matched-frequency/3.0.0 +I0913 17:37:26.357086 140507706811584 reader.py:262] Creating a tf.data.Dataset reading 16 files located in folders: /data/imagenet/pytorch/imagenet_v2/matched-frequency/3.0.0. +I0913 17:37:26.467871 140507706811584 logging_logger.py:49] Constructing tf.data.Dataset imagenet_v2 for split test, from /data/imagenet/pytorch/imagenet_v2/matched-frequency/3.0.0 +I0913 17:44:02.784277 140507706811584 submission_runner.py:516] Time since start: 644.76s, Step: 1, {'train/accuracy': 0.0023046875, 'train/loss': 6.90775634765625, 'validation/accuracy': 0.00192, 'validation/loss': 6.90775625, 'validation/num_examples': 50000, 'test/accuracy': 0.0019, 'test/loss': 6.90775546875, 'test/num_examples': 10000, 'score': 95.80058550834656, 'total_duration': 644.7618706226349, 'accumulated_submission_time': 95.80058550834656, 'accumulated_eval_time': 548.3675837516785, 'accumulated_logging_time': 0} +I0913 17:44:02.819183 140452846106368 logging_writer.py:48] [1] accumulated_eval_time=548.368, accumulated_logging_time=0, accumulated_submission_time=95.8006, global_step=1, preemption_count=0, score=95.8006, test/accuracy=0.0019, test/loss=6.90776, test/num_examples=10000, total_duration=644.762, train/accuracy=0.00230469, train/loss=6.90776, validation/accuracy=0.00192, validation/loss=6.90776, validation/num_examples=50000 +I0913 17:44:04.134599 140452837713664 logging_writer.py:48] [1] global_step=1, grad_norm=0.362709, loss=6.90776 +I0913 17:44:04.137554 140507706811584 submission.py:307] 1) loss = 6.908, grad_norm = 0.363 +I0913 17:44:04.377089 140452846106368 logging_writer.py:48] [2] global_step=2, grad_norm=0.361491, loss=6.90775 +I0913 17:44:04.381827 140507706811584 submission.py:307] 2) loss = 6.908, grad_norm = 0.361 +I0913 17:44:04.624587 140452837713664 logging_writer.py:48] [3] global_step=3, grad_norm=0.368323, loss=6.90775 +I0913 17:44:04.629356 140507706811584 submission.py:307] 3) loss = 6.908, grad_norm = 0.368 +I0913 17:44:04.859891 140452846106368 logging_writer.py:48] [4] global_step=4, grad_norm=0.365549, loss=6.90775 +I0913 17:44:04.864735 140507706811584 submission.py:307] 4) loss = 6.908, grad_norm = 0.366 +I0913 17:44:05.120210 140452837713664 logging_writer.py:48] [5] global_step=5, grad_norm=0.368308, loss=6.90775 +I0913 17:44:05.124001 140507706811584 submission.py:307] 5) loss = 6.908, grad_norm = 0.368 +I0913 17:44:05.347893 140452846106368 logging_writer.py:48] [6] global_step=6, grad_norm=0.370455, loss=6.90775 +I0913 17:44:05.353224 140507706811584 submission.py:307] 6) loss = 6.908, grad_norm = 0.370 +I0913 17:44:05.609487 140452837713664 logging_writer.py:48] [7] global_step=7, grad_norm=0.375421, loss=6.90775 +I0913 17:44:05.612707 140507706811584 submission.py:307] 7) loss = 6.908, grad_norm = 0.375 +I0913 17:44:05.832950 140452846106368 logging_writer.py:48] [8] global_step=8, grad_norm=0.364655, loss=6.90775 +I0913 17:44:05.841862 140507706811584 submission.py:307] 8) loss = 6.908, grad_norm = 0.365 +I0913 17:44:06.124198 140452837713664 logging_writer.py:48] [9] global_step=9, grad_norm=0.370484, loss=6.90775 +I0913 17:44:06.139661 140507706811584 submission.py:307] 9) loss = 6.908, grad_norm = 0.370 +I0913 17:44:06.412063 140452846106368 logging_writer.py:48] [10] global_step=10, grad_norm=0.374877, loss=6.90773 +I0913 17:44:06.416011 140507706811584 submission.py:307] 10) loss = 6.908, grad_norm = 0.375 +I0913 17:44:06.664301 140452837713664 logging_writer.py:48] [11] global_step=11, grad_norm=0.37034, loss=6.90773 +I0913 17:44:06.668539 140507706811584 submission.py:307] 11) loss = 6.908, grad_norm = 0.370 +I0913 17:44:06.973272 140452846106368 logging_writer.py:48] [12] global_step=12, grad_norm=0.353503, loss=6.90775 +I0913 17:44:07.005450 140507706811584 submission.py:307] 12) loss = 6.908, grad_norm = 0.354 +I0913 17:44:07.349489 140452837713664 logging_writer.py:48] [13] global_step=13, grad_norm=0.35578, loss=6.90773 +I0913 17:44:07.370834 140507706811584 submission.py:307] 13) loss = 6.908, grad_norm = 0.356 +I0913 17:44:07.685078 140452846106368 logging_writer.py:48] [14] global_step=14, grad_norm=0.360333, loss=6.90774 +I0913 17:44:07.693747 140507706811584 submission.py:307] 14) loss = 6.908, grad_norm = 0.360 +I0913 17:44:08.081029 140452837713664 logging_writer.py:48] [15] global_step=15, grad_norm=0.358725, loss=6.90773 +I0913 17:44:08.094138 140507706811584 submission.py:307] 15) loss = 6.908, grad_norm = 0.359 +I0913 17:44:08.681700 140452846106368 logging_writer.py:48] [16] global_step=16, grad_norm=0.349947, loss=6.90774 +I0913 17:44:08.718491 140507706811584 submission.py:307] 16) loss = 6.908, grad_norm = 0.350 +I0913 17:44:09.045643 140452837713664 logging_writer.py:48] [17] global_step=17, grad_norm=0.352708, loss=6.90771 +I0913 17:44:09.063006 140507706811584 submission.py:307] 17) loss = 6.908, grad_norm = 0.353 +I0913 17:44:09.536669 140452846106368 logging_writer.py:48] [18] global_step=18, grad_norm=0.361763, loss=6.90774 +I0913 17:44:09.550876 140507706811584 submission.py:307] 18) loss = 6.908, grad_norm = 0.362 +I0913 17:44:09.943442 140452837713664 logging_writer.py:48] [19] global_step=19, grad_norm=0.367643, loss=6.90771 +I0913 17:44:09.946783 140507706811584 submission.py:307] 19) loss = 6.908, grad_norm = 0.368 +I0913 17:44:10.184718 140452846106368 logging_writer.py:48] [20] global_step=20, grad_norm=0.374705, loss=6.90769 +I0913 17:44:10.187698 140507706811584 submission.py:307] 20) loss = 6.908, grad_norm = 0.375 +I0913 17:44:10.565681 140452837713664 logging_writer.py:48] [21] global_step=21, grad_norm=0.368225, loss=6.90766 +I0913 17:44:10.668308 140507706811584 submission.py:307] 21) loss = 6.908, grad_norm = 0.368 +I0913 17:44:11.114371 140452846106368 logging_writer.py:48] [22] global_step=22, grad_norm=0.375891, loss=6.90767 +I0913 17:44:11.163539 140507706811584 submission.py:307] 22) loss = 6.908, grad_norm = 0.376 +I0913 17:44:11.540042 140452837713664 logging_writer.py:48] [23] global_step=23, grad_norm=0.368491, loss=6.90765 +I0913 17:44:11.575742 140507706811584 submission.py:307] 23) loss = 6.908, grad_norm = 0.368 +I0913 17:44:11.963864 140452846106368 logging_writer.py:48] [24] global_step=24, grad_norm=0.374033, loss=6.90764 +I0913 17:44:11.968832 140507706811584 submission.py:307] 24) loss = 6.908, grad_norm = 0.374 +I0913 17:44:12.215263 140452837713664 logging_writer.py:48] [25] global_step=25, grad_norm=0.367103, loss=6.90759 +I0913 17:44:12.220545 140507706811584 submission.py:307] 25) loss = 6.908, grad_norm = 0.367 +I0913 17:44:12.454952 140452846106368 logging_writer.py:48] [26] global_step=26, grad_norm=0.357182, loss=6.90763 +I0913 17:44:12.458772 140507706811584 submission.py:307] 26) loss = 6.908, grad_norm = 0.357 +I0913 17:44:12.690603 140452837713664 logging_writer.py:48] [27] global_step=27, grad_norm=0.371009, loss=6.90764 +I0913 17:44:12.694466 140507706811584 submission.py:307] 27) loss = 6.908, grad_norm = 0.371 +I0913 17:44:12.977493 140452846106368 logging_writer.py:48] [28] global_step=28, grad_norm=0.37503, loss=6.90759 +I0913 17:44:13.016875 140507706811584 submission.py:307] 28) loss = 6.908, grad_norm = 0.375 +I0913 17:44:13.507184 140452837713664 logging_writer.py:48] [29] global_step=29, grad_norm=0.364762, loss=6.90768 +I0913 17:44:13.517848 140507706811584 submission.py:307] 29) loss = 6.908, grad_norm = 0.365 +I0913 17:44:14.077676 140452846106368 logging_writer.py:48] [30] global_step=30, grad_norm=0.360269, loss=6.90755 +I0913 17:44:14.097771 140507706811584 submission.py:307] 30) loss = 6.908, grad_norm = 0.360 +I0913 17:44:14.492542 140452837713664 logging_writer.py:48] [31] global_step=31, grad_norm=0.372131, loss=6.9076 +I0913 17:44:14.496263 140507706811584 submission.py:307] 31) loss = 6.908, grad_norm = 0.372 +I0913 17:44:14.733566 140452846106368 logging_writer.py:48] [32] global_step=32, grad_norm=0.36798, loss=6.90756 +I0913 17:44:14.749820 140507706811584 submission.py:307] 32) loss = 6.908, grad_norm = 0.368 +I0913 17:44:15.237492 140452837713664 logging_writer.py:48] [33] global_step=33, grad_norm=0.38238, loss=6.90738 +I0913 17:44:15.266777 140507706811584 submission.py:307] 33) loss = 6.907, grad_norm = 0.382 +I0913 17:44:15.833511 140452846106368 logging_writer.py:48] [34] global_step=34, grad_norm=0.352205, loss=6.90759 +I0913 17:44:15.857507 140507706811584 submission.py:307] 34) loss = 6.908, grad_norm = 0.352 +I0913 17:44:16.309959 140452837713664 logging_writer.py:48] [35] global_step=35, grad_norm=0.373655, loss=6.90741 +I0913 17:44:16.314181 140507706811584 submission.py:307] 35) loss = 6.907, grad_norm = 0.374 +I0913 17:44:16.547004 140452846106368 logging_writer.py:48] [36] global_step=36, grad_norm=0.377812, loss=6.90741 +I0913 17:44:16.552094 140507706811584 submission.py:307] 36) loss = 6.907, grad_norm = 0.378 +I0913 17:44:16.893793 140452837713664 logging_writer.py:48] [37] global_step=37, grad_norm=0.348697, loss=6.90745 +I0913 17:44:16.936952 140507706811584 submission.py:307] 37) loss = 6.907, grad_norm = 0.349 +I0913 17:44:17.325618 140452846106368 logging_writer.py:48] [38] global_step=38, grad_norm=0.374552, loss=6.90737 +I0913 17:44:17.363870 140507706811584 submission.py:307] 38) loss = 6.907, grad_norm = 0.375 +I0913 17:44:18.000277 140452837713664 logging_writer.py:48] [39] global_step=39, grad_norm=0.385815, loss=6.90736 +I0913 17:44:18.084770 140507706811584 submission.py:307] 39) loss = 6.907, grad_norm = 0.386 +I0913 17:44:18.423692 140452846106368 logging_writer.py:48] [40] global_step=40, grad_norm=0.353447, loss=6.90736 +I0913 17:44:18.427528 140507706811584 submission.py:307] 40) loss = 6.907, grad_norm = 0.353 +I0913 17:44:18.655935 140452837713664 logging_writer.py:48] [41] global_step=41, grad_norm=0.386844, loss=6.90719 +I0913 17:44:18.658906 140507706811584 submission.py:307] 41) loss = 6.907, grad_norm = 0.387 +I0913 17:44:18.898946 140452846106368 logging_writer.py:48] [42] global_step=42, grad_norm=0.381523, loss=6.90726 +I0913 17:44:18.902685 140507706811584 submission.py:307] 42) loss = 6.907, grad_norm = 0.382 +I0913 17:44:19.157806 140452837713664 logging_writer.py:48] [43] global_step=43, grad_norm=0.345086, loss=6.90727 +I0913 17:44:19.186169 140507706811584 submission.py:307] 43) loss = 6.907, grad_norm = 0.345 +I0913 17:44:19.779944 140452846106368 logging_writer.py:48] [44] global_step=44, grad_norm=0.384758, loss=6.90696 +I0913 17:44:19.791590 140507706811584 submission.py:307] 44) loss = 6.907, grad_norm = 0.385 +I0913 17:44:20.356604 140452846106368 logging_writer.py:48] [45] global_step=45, grad_norm=0.350518, loss=6.90723 +I0913 17:44:20.393217 140507706811584 submission.py:307] 45) loss = 6.907, grad_norm = 0.351 +I0913 17:44:20.664354 140452837713664 logging_writer.py:48] [46] global_step=46, grad_norm=0.391739, loss=6.90696 +I0913 17:44:20.667399 140507706811584 submission.py:307] 46) loss = 6.907, grad_norm = 0.392 +I0913 17:44:20.904763 140452846106368 logging_writer.py:48] [47] global_step=47, grad_norm=0.344412, loss=6.90716 +I0913 17:44:20.907819 140507706811584 submission.py:307] 47) loss = 6.907, grad_norm = 0.344 +I0913 17:44:21.164598 140452837713664 logging_writer.py:48] [48] global_step=48, grad_norm=0.399498, loss=6.90671 +I0913 17:44:21.168735 140507706811584 submission.py:307] 48) loss = 6.907, grad_norm = 0.399 +I0913 17:44:21.409513 140452846106368 logging_writer.py:48] [49] global_step=49, grad_norm=0.349123, loss=6.90708 +I0913 17:44:21.416832 140507706811584 submission.py:307] 49) loss = 6.907, grad_norm = 0.349 +I0913 17:44:22.066917 140452837713664 logging_writer.py:48] [50] global_step=50, grad_norm=0.352266, loss=6.90712 +I0913 17:44:22.109349 140507706811584 submission.py:307] 50) loss = 6.907, grad_norm = 0.352 +I0913 17:44:22.613513 140452846106368 logging_writer.py:48] [51] global_step=51, grad_norm=0.408271, loss=6.9064 +I0913 17:44:22.630155 140507706811584 submission.py:307] 51) loss = 6.906, grad_norm = 0.408 +I0913 17:44:23.117403 140452837713664 logging_writer.py:48] [52] global_step=52, grad_norm=0.391871, loss=6.90658 +I0913 17:44:23.121976 140507706811584 submission.py:307] 52) loss = 6.907, grad_norm = 0.392 +I0913 17:44:23.357421 140452846106368 logging_writer.py:48] [53] global_step=53, grad_norm=0.395012, loss=6.90646 +I0913 17:44:23.360462 140507706811584 submission.py:307] 53) loss = 6.906, grad_norm = 0.395 +I0913 17:44:23.591939 140452837713664 logging_writer.py:48] [54] global_step=54, grad_norm=0.415236, loss=6.90623 +I0913 17:44:23.596113 140507706811584 submission.py:307] 54) loss = 6.906, grad_norm = 0.415 +I0913 17:44:23.820769 140452846106368 logging_writer.py:48] [55] global_step=55, grad_norm=0.396556, loss=6.90623 +I0913 17:44:23.824333 140507706811584 submission.py:307] 55) loss = 6.906, grad_norm = 0.397 +I0913 17:44:24.105881 140452837713664 logging_writer.py:48] [56] global_step=56, grad_norm=0.414558, loss=6.90609 +I0913 17:44:24.190201 140507706811584 submission.py:307] 56) loss = 6.906, grad_norm = 0.415 +I0913 17:44:24.885396 140452846106368 logging_writer.py:48] [57] global_step=57, grad_norm=0.428649, loss=6.90628 +I0913 17:44:24.927341 140507706811584 submission.py:307] 57) loss = 6.906, grad_norm = 0.429 +I0913 17:44:25.415839 140452837713664 logging_writer.py:48] [58] global_step=58, grad_norm=0.418884, loss=6.90626 +I0913 17:44:25.420171 140507706811584 submission.py:307] 58) loss = 6.906, grad_norm = 0.419 +I0913 17:44:25.634916 140452846106368 logging_writer.py:48] [59] global_step=59, grad_norm=0.410458, loss=6.90564 +I0913 17:44:25.638183 140507706811584 submission.py:307] 59) loss = 6.906, grad_norm = 0.410 +I0913 17:44:25.920754 140452837713664 logging_writer.py:48] [60] global_step=60, grad_norm=0.430534, loss=6.90533 +I0913 17:44:25.923713 140507706811584 submission.py:307] 60) loss = 6.905, grad_norm = 0.431 +I0913 17:44:26.163055 140452846106368 logging_writer.py:48] [61] global_step=61, grad_norm=0.368911, loss=6.90603 +I0913 17:44:26.167656 140507706811584 submission.py:307] 61) loss = 6.906, grad_norm = 0.369 +I0913 17:44:26.593507 140452837713664 logging_writer.py:48] [62] global_step=62, grad_norm=0.422653, loss=6.90534 +I0913 17:44:26.610764 140507706811584 submission.py:307] 62) loss = 6.905, grad_norm = 0.423 +I0913 17:44:27.283428 140452846106368 logging_writer.py:48] [63] global_step=63, grad_norm=0.376899, loss=6.90598 +I0913 17:44:27.330046 140507706811584 submission.py:307] 63) loss = 6.906, grad_norm = 0.377 +I0913 17:44:27.771028 140452837713664 logging_writer.py:48] [64] global_step=64, grad_norm=0.401941, loss=6.90569 +I0913 17:44:27.774251 140507706811584 submission.py:307] 64) loss = 6.906, grad_norm = 0.402 +I0913 17:44:28.019049 140452846106368 logging_writer.py:48] [65] global_step=65, grad_norm=0.435768, loss=6.90487 +I0913 17:44:28.022039 140507706811584 submission.py:307] 65) loss = 6.905, grad_norm = 0.436 +I0913 17:44:28.248074 140452837713664 logging_writer.py:48] [66] global_step=66, grad_norm=0.388623, loss=6.90495 +I0913 17:44:28.251782 140507706811584 submission.py:307] 66) loss = 6.905, grad_norm = 0.389 +I0913 17:44:28.487314 140452846106368 logging_writer.py:48] [67] global_step=67, grad_norm=0.441107, loss=6.90476 +I0913 17:44:28.491402 140507706811584 submission.py:307] 67) loss = 6.905, grad_norm = 0.441 +I0913 17:44:28.920637 140452837713664 logging_writer.py:48] [68] global_step=68, grad_norm=0.451474, loss=6.90534 +I0913 17:44:28.951344 140507706811584 submission.py:307] 68) loss = 6.905, grad_norm = 0.451 +I0913 17:44:29.549412 140452846106368 logging_writer.py:48] [69] global_step=69, grad_norm=0.392044, loss=6.90433 +I0913 17:44:29.585914 140507706811584 submission.py:307] 69) loss = 6.904, grad_norm = 0.392 +I0913 17:44:30.081146 140452837713664 logging_writer.py:48] [70] global_step=70, grad_norm=0.445172, loss=6.90434 +I0913 17:44:30.085824 140507706811584 submission.py:307] 70) loss = 6.904, grad_norm = 0.445 +I0913 17:44:30.339871 140452846106368 logging_writer.py:48] [71] global_step=71, grad_norm=0.451645, loss=6.9044 +I0913 17:44:30.342977 140507706811584 submission.py:307] 71) loss = 6.904, grad_norm = 0.452 +I0913 17:44:30.563396 140452837713664 logging_writer.py:48] [72] global_step=72, grad_norm=0.44954, loss=6.90427 +I0913 17:44:30.566409 140507706811584 submission.py:307] 72) loss = 6.904, grad_norm = 0.450 +I0913 17:44:30.795397 140452846106368 logging_writer.py:48] [73] global_step=73, grad_norm=0.408346, loss=6.9042 +I0913 17:44:30.800322 140507706811584 submission.py:307] 73) loss = 6.904, grad_norm = 0.408 +I0913 17:44:31.184462 140452837713664 logging_writer.py:48] [74] global_step=74, grad_norm=0.386298, loss=6.90462 +I0913 17:44:31.247664 140507706811584 submission.py:307] 74) loss = 6.905, grad_norm = 0.386 +I0913 17:44:31.817857 140452846106368 logging_writer.py:48] [75] global_step=75, grad_norm=0.445807, loss=6.90287 +I0913 17:44:31.837181 140507706811584 submission.py:307] 75) loss = 6.903, grad_norm = 0.446 +I0913 17:44:32.477140 140452837713664 logging_writer.py:48] [76] global_step=76, grad_norm=0.417021, loss=6.90318 +I0913 17:44:32.480611 140507706811584 submission.py:307] 76) loss = 6.903, grad_norm = 0.417 +I0913 17:44:32.711183 140452846106368 logging_writer.py:48] [77] global_step=77, grad_norm=0.451657, loss=6.90284 +I0913 17:44:32.714190 140507706811584 submission.py:307] 77) loss = 6.903, grad_norm = 0.452 +I0913 17:44:32.976002 140452837713664 logging_writer.py:48] [78] global_step=78, grad_norm=0.46583, loss=6.90381 +I0913 17:44:32.978985 140507706811584 submission.py:307] 78) loss = 6.904, grad_norm = 0.466 +I0913 17:44:33.250414 140452846106368 logging_writer.py:48] [79] global_step=79, grad_norm=0.435857, loss=6.90331 +I0913 17:44:33.258174 140507706811584 submission.py:307] 79) loss = 6.903, grad_norm = 0.436 +I0913 17:44:33.830071 140452837713664 logging_writer.py:48] [80] global_step=80, grad_norm=0.478337, loss=6.90168 +I0913 17:44:33.895724 140507706811584 submission.py:307] 80) loss = 6.902, grad_norm = 0.478 +I0913 17:44:34.637379 140452846106368 logging_writer.py:48] [81] global_step=81, grad_norm=0.420846, loss=6.90295 +I0913 17:44:34.672922 140507706811584 submission.py:307] 81) loss = 6.903, grad_norm = 0.421 +I0913 17:44:34.998379 140452837713664 logging_writer.py:48] [82] global_step=82, grad_norm=0.474566, loss=6.90114 +I0913 17:44:35.001483 140507706811584 submission.py:307] 82) loss = 6.901, grad_norm = 0.475 +I0913 17:44:35.217116 140452846106368 logging_writer.py:48] [83] global_step=83, grad_norm=0.467234, loss=6.90087 +I0913 17:44:35.220120 140507706811584 submission.py:307] 83) loss = 6.901, grad_norm = 0.467 +I0913 17:44:35.463185 140452837713664 logging_writer.py:48] [84] global_step=84, grad_norm=0.413481, loss=6.90384 +I0913 17:44:35.466207 140507706811584 submission.py:307] 84) loss = 6.904, grad_norm = 0.413 +I0913 17:44:35.781471 140452846106368 logging_writer.py:48] [85] global_step=85, grad_norm=0.486484, loss=6.90059 +I0913 17:44:35.810763 140507706811584 submission.py:307] 85) loss = 6.901, grad_norm = 0.486 +I0913 17:44:36.317809 140452837713664 logging_writer.py:48] [86] global_step=86, grad_norm=0.394523, loss=6.9035 +I0913 17:44:36.381934 140507706811584 submission.py:307] 86) loss = 6.903, grad_norm = 0.395 +I0913 17:44:36.896728 140452846106368 logging_writer.py:48] [87] global_step=87, grad_norm=0.497957, loss=6.90067 +I0913 17:44:36.926704 140507706811584 submission.py:307] 87) loss = 6.901, grad_norm = 0.498 +I0913 17:44:37.233686 140452837713664 logging_writer.py:48] [88] global_step=88, grad_norm=0.414269, loss=6.9023 +I0913 17:44:37.237026 140507706811584 submission.py:307] 88) loss = 6.902, grad_norm = 0.414 +I0913 17:44:37.494848 140452846106368 logging_writer.py:48] [89] global_step=89, grad_norm=0.486093, loss=6.90043 +I0913 17:44:37.497877 140507706811584 submission.py:307] 89) loss = 6.900, grad_norm = 0.486 +I0913 17:44:37.747021 140452837713664 logging_writer.py:48] [90] global_step=90, grad_norm=0.437461, loss=6.90123 +I0913 17:44:37.755734 140507706811584 submission.py:307] 90) loss = 6.901, grad_norm = 0.437 +I0913 17:44:38.009491 140452846106368 logging_writer.py:48] [91] global_step=91, grad_norm=0.391668, loss=6.90228 +I0913 17:44:38.039232 140507706811584 submission.py:307] 91) loss = 6.902, grad_norm = 0.392 +I0913 17:44:38.405481 140452837713664 logging_writer.py:48] [92] global_step=92, grad_norm=0.495325, loss=6.8982 +I0913 17:44:38.442750 140507706811584 submission.py:307] 92) loss = 6.898, grad_norm = 0.495 +I0913 17:44:38.917698 140452846106368 logging_writer.py:48] [93] global_step=93, grad_norm=0.438118, loss=6.90048 +I0913 17:44:38.952540 140507706811584 submission.py:307] 93) loss = 6.900, grad_norm = 0.438 +I0913 17:44:39.464607 140452837713664 logging_writer.py:48] [94] global_step=94, grad_norm=0.488803, loss=6.89998 +I0913 17:44:39.468547 140507706811584 submission.py:307] 94) loss = 6.900, grad_norm = 0.489 +I0913 17:44:39.705710 140452846106368 logging_writer.py:48] [95] global_step=95, grad_norm=0.461871, loss=6.89803 +I0913 17:44:39.708755 140507706811584 submission.py:307] 95) loss = 6.898, grad_norm = 0.462 +I0913 17:44:39.973504 140452837713664 logging_writer.py:48] [96] global_step=96, grad_norm=0.413842, loss=6.9023 +I0913 17:44:39.995272 140507706811584 submission.py:307] 96) loss = 6.902, grad_norm = 0.414 +I0913 17:44:40.605496 140452846106368 logging_writer.py:48] [97] global_step=97, grad_norm=0.499999, loss=6.89986 +I0913 17:44:40.622224 140507706811584 submission.py:307] 97) loss = 6.900, grad_norm = 0.500 +I0913 17:44:41.139040 140452837713664 logging_writer.py:48] [98] global_step=98, grad_norm=0.484703, loss=6.89734 +I0913 17:44:41.186374 140507706811584 submission.py:307] 98) loss = 6.897, grad_norm = 0.485 +I0913 17:44:41.529788 140452846106368 logging_writer.py:48] [99] global_step=99, grad_norm=0.499999, loss=6.89657 +I0913 17:44:41.533914 140507706811584 submission.py:307] 99) loss = 6.897, grad_norm = 0.500 +I0913 17:44:41.837475 140452837713664 logging_writer.py:48] [100] global_step=100, grad_norm=0.48935, loss=6.89645 +I0913 17:44:41.844575 140507706811584 submission.py:307] 100) loss = 6.896, grad_norm = 0.489 +I0913 17:48:19.608519 140452846106368 logging_writer.py:48] [500] global_step=500, grad_norm=0.499999, loss=6.75496 +I0913 17:48:19.612261 140507706811584 submission.py:307] 500) loss = 6.755, grad_norm = 0.500 +I0913 17:55:08.047214 140452837713664 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.5, loss=6.60979 +I0913 17:55:08.051046 140507706811584 submission.py:307] 1000) loss = 6.610, grad_norm = 0.500 +I0913 18:00:57.649372 140452846106368 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.5, loss=6.1776 +I0913 18:00:57.714872 140507706811584 submission.py:307] 1500) loss = 6.178, grad_norm = 0.500 +I0913 18:05:47.690502 140452837713664 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.5, loss=5.92444 +I0913 18:05:47.696420 140507706811584 submission.py:307] 2000) loss = 5.924, grad_norm = 0.500 +I0913 18:13:39.137195 140452846106368 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.5, loss=5.80865 +I0913 18:13:39.140805 140507706811584 submission.py:307] 2500) loss = 5.809, grad_norm = 0.500 +I0913 18:17:39.055826 140452837713664 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.5, loss=5.58836 +I0913 18:17:39.060024 140507706811584 submission.py:307] 3000) loss = 5.588, grad_norm = 0.500 +I0913 18:24:38.554202 140452846106368 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.5, loss=5.65641 +I0913 18:24:38.558060 140507706811584 submission.py:307] 3500) loss = 5.656, grad_norm = 0.500 +I0913 18:27:07.402356 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 18:28:23.343933 140507706811584 spec.py:346] Evaluating on the validation split. +I0913 18:29:28.856454 140507706811584 spec.py:363] Evaluating on the test split. +I0913 18:29:31.445126 140507706811584 submission_runner.py:516] Time since start: 3373.42s, Step: 3623, {'train/accuracy': 0.09013671875, 'train/loss': 5.483372802734375, 'validation/accuracy': 0.08084, 'validation/loss': 5.48525625, 'validation/num_examples': 50000, 'test/accuracy': 0.0754, 'test/loss': 5.2725359375, 'test/num_examples': 10000, 'score': 2665.0133833885193, 'total_duration': 3373.4229576587677, 'accumulated_submission_time': 2665.0133833885193, 'accumulated_eval_time': 692.409998178482, 'accumulated_logging_time': 0.04683208465576172} +I0913 18:29:31.783987 140444834858752 logging_writer.py:48] [3623] accumulated_eval_time=692.41, accumulated_logging_time=0.0468321, accumulated_submission_time=2665.01, global_step=3623, preemption_count=0, score=2665.01, test/accuracy=0.0754, test/loss=5.27254, test/num_examples=10000, total_duration=3373.42, train/accuracy=0.0901367, train/loss=5.48337, validation/accuracy=0.08084, validation/loss=5.48526, validation/num_examples=50000 +I0913 18:33:50.675435 140444843251456 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.5, loss=5.47092 +I0913 18:33:50.680538 140507706811584 submission.py:307] 4000) loss = 5.471, grad_norm = 0.500 +I0913 18:40:20.723733 140444834858752 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.5, loss=4.94291 +I0913 18:40:20.727577 140507706811584 submission.py:307] 4500) loss = 4.943, grad_norm = 0.500 +I0913 18:48:43.011739 140444843251456 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.5, loss=4.83837 +I0913 18:48:43.024516 140507706811584 submission.py:307] 5000) loss = 4.838, grad_norm = 0.500 +I0913 18:52:49.473554 140444834858752 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.5, loss=5.04933 +I0913 18:52:49.480227 140507706811584 submission.py:307] 5500) loss = 5.049, grad_norm = 0.500 +I0913 19:00:16.504827 140444843251456 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.5, loss=5.29933 +I0913 19:00:16.508623 140507706811584 submission.py:307] 6000) loss = 5.299, grad_norm = 0.500 +I0913 19:06:48.869977 140444834858752 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.5, loss=4.73225 +I0913 19:06:48.874665 140507706811584 submission.py:307] 6500) loss = 4.732, grad_norm = 0.500 +I0913 19:12:05.025152 140444843251456 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.499999, loss=5.9351 +I0913 19:12:05.029351 140507706811584 submission.py:307] 7000) loss = 5.935, grad_norm = 0.500 +I0913 19:12:27.396579 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 19:13:42.855057 140507706811584 spec.py:346] Evaluating on the validation split. +I0913 19:14:47.940235 140507706811584 spec.py:363] Evaluating on the test split. +I0913 19:14:48.903311 140507706811584 submission_runner.py:516] Time since start: 6090.88s, Step: 7023, {'train/accuracy': 0.25580078125, 'train/loss': 4.329656372070312, 'validation/accuracy': 0.21272, 'validation/loss': 4.58559375, 'validation/num_examples': 50000, 'test/accuracy': 0.2128, 'test/loss': 4.00091328125, 'test/num_examples': 10000, 'score': 5230.912830114365, 'total_duration': 6090.881366968155, 'accumulated_submission_time': 5230.912830114365, 'accumulated_eval_time': 833.9168169498444, 'accumulated_logging_time': 0.3999044895172119} +I0913 19:14:49.228518 140484912580352 logging_writer.py:48] [7023] accumulated_eval_time=833.917, accumulated_logging_time=0.399904, accumulated_submission_time=5230.91, global_step=7023, preemption_count=0, score=5230.91, test/accuracy=0.2128, test/loss=4.00091, test/num_examples=10000, total_duration=6090.88, train/accuracy=0.255801, train/loss=4.32966, validation/accuracy=0.21272, validation/loss=4.58559, validation/num_examples=50000 +I0913 19:22:19.931613 140485125469952 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.5, loss=4.03787 +I0913 19:22:19.935435 140507706811584 submission.py:307] 7500) loss = 4.038, grad_norm = 0.500 +I0913 19:27:07.023656 140484912580352 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.5, loss=5.49669 +I0913 19:27:07.029228 140507706811584 submission.py:307] 8000) loss = 5.497, grad_norm = 0.500 +I0913 19:34:23.171901 140485125469952 logging_writer.py:48] [8500] global_step=8500, grad_norm=0.5, loss=5.09999 +I0913 19:34:23.177913 140507706811584 submission.py:307] 8500) loss = 5.100, grad_norm = 0.500 +I0913 19:40:49.424042 140484912580352 logging_writer.py:48] [9000] global_step=9000, grad_norm=0.5, loss=5.1985 +I0913 19:40:49.430588 140507706811584 submission.py:307] 9000) loss = 5.198, grad_norm = 0.500 +I0913 19:46:08.895958 140485125469952 logging_writer.py:48] [9500] global_step=9500, grad_norm=0.5, loss=3.75837 +I0913 19:46:08.901148 140507706811584 submission.py:307] 9500) loss = 3.758, grad_norm = 0.500 +I0913 19:54:36.534573 140484912580352 logging_writer.py:48] [10000] global_step=10000, grad_norm=0.499999, loss=5.69358 +I0913 19:54:36.538213 140507706811584 submission.py:307] 10000) loss = 5.694, grad_norm = 0.500 +I0913 19:57:44.447486 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 19:58:54.492428 140507706811584 spec.py:346] Evaluating on the validation split. +I0913 20:00:00.238492 140507706811584 spec.py:363] Evaluating on the test split. +I0913 20:00:01.169079 140507706811584 submission_runner.py:516] Time since start: 8803.15s, Step: 10375, {'train/accuracy': 0.37193359375, 'train/loss': 3.657047119140625, 'validation/accuracy': 0.31606, 'validation/loss': 4.042581875, 'validation/num_examples': 50000, 'test/accuracy': 0.313, 'test/loss': 3.3613, 'test/num_examples': 10000, 'score': 7797.297001361847, 'total_duration': 8803.147234916687, 'accumulated_submission_time': 7797.297001361847, 'accumulated_eval_time': 970.6382474899292, 'accumulated_logging_time': 0.7354223728179932} +I0913 20:00:01.588235 140485049935616 logging_writer.py:48] [10375] accumulated_eval_time=970.638, accumulated_logging_time=0.735422, accumulated_submission_time=7797.3, global_step=10375, preemption_count=0, score=7797.3, test/accuracy=0.313, test/loss=3.3613, test/num_examples=10000, total_duration=8803.15, train/accuracy=0.371934, train/loss=3.65705, validation/accuracy=0.31606, validation/loss=4.04258, validation/num_examples=50000 +I0913 20:01:04.421160 140485117077248 logging_writer.py:48] [10500] global_step=10500, grad_norm=0.5, loss=4.34967 +I0913 20:01:04.424932 140507706811584 submission.py:307] 10500) loss = 4.350, grad_norm = 0.500 +I0913 20:09:42.373429 140485049935616 logging_writer.py:48] [11000] global_step=11000, grad_norm=0.5, loss=3.42539 +I0913 20:09:42.377252 140507706811584 submission.py:307] 11000) loss = 3.425, grad_norm = 0.500 +I0913 20:16:27.355471 140485117077248 logging_writer.py:48] [11500] global_step=11500, grad_norm=0.5, loss=3.14633 +I0913 20:16:27.389374 140507706811584 submission.py:307] 11500) loss = 3.146, grad_norm = 0.500 +I0913 20:21:58.007006 140485049935616 logging_writer.py:48] [12000] global_step=12000, grad_norm=0.5, loss=3.57017 +I0913 20:21:58.011918 140507706811584 submission.py:307] 12000) loss = 3.570, grad_norm = 0.500 +I0913 20:30:38.363394 140485117077248 logging_writer.py:48] [12500] global_step=12500, grad_norm=0.5, loss=3.18701 +I0913 20:30:38.367917 140507706811584 submission.py:307] 12500) loss = 3.187, grad_norm = 0.500 +I0913 20:35:00.273119 140485049935616 logging_writer.py:48] [13000] global_step=13000, grad_norm=0.5, loss=3.21071 +I0913 20:35:00.279783 140507706811584 submission.py:307] 13000) loss = 3.211, grad_norm = 0.500 +I0913 20:42:16.268245 140485117077248 logging_writer.py:48] [13500] global_step=13500, grad_norm=0.5, loss=3.0089 +I0913 20:42:16.273520 140507706811584 submission.py:307] 13500) loss = 3.009, grad_norm = 0.500 +I0913 20:43:01.721394 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 20:44:14.801697 140507706811584 spec.py:346] Evaluating on the validation split. +I0913 20:45:19.735058 140507706811584 spec.py:363] Evaluating on the test split. +I0913 20:45:20.703338 140507706811584 submission_runner.py:516] Time since start: 11522.68s, Step: 13533, {'train/accuracy': 0.451796875, 'train/loss': 3.357933349609375, 'validation/accuracy': 0.38034, 'validation/loss': 3.827965625, 'validation/num_examples': 50000, 'test/accuracy': 0.3787, 'test/loss': 2.8829498046875, 'test/num_examples': 10000, 'score': 10368.7392745018, 'total_duration': 11522.681096076965, 'accumulated_submission_time': 10368.7392745018, 'accumulated_eval_time': 1109.6197061538696, 'accumulated_logging_time': 1.2692897319793701} +I0913 20:45:21.092357 140485033150208 logging_writer.py:48] [13533] accumulated_eval_time=1109.62, accumulated_logging_time=1.26929, accumulated_submission_time=10368.7, global_step=13533, preemption_count=0, score=10368.7, test/accuracy=0.3787, test/loss=2.88295, test/num_examples=10000, total_duration=11522.7, train/accuracy=0.451797, train/loss=3.35793, validation/accuracy=0.38034, validation/loss=3.82797, validation/num_examples=50000 +I0913 20:51:03.816929 140485083506432 logging_writer.py:48] [14000] global_step=14000, grad_norm=0.5, loss=3.0414 +I0913 20:51:03.821037 140507706811584 submission.py:307] 14000) loss = 3.041, grad_norm = 0.500 +I0913 20:57:08.684439 140485033150208 logging_writer.py:48] [14500] global_step=14500, grad_norm=0.5, loss=3.59709 +I0913 20:57:08.688704 140507706811584 submission.py:307] 14500) loss = 3.597, grad_norm = 0.500 +I0913 21:05:36.338385 140485083506432 logging_writer.py:48] [15000] global_step=15000, grad_norm=0.5, loss=2.83885 +I0913 21:05:36.342253 140507706811584 submission.py:307] 15000) loss = 2.839, grad_norm = 0.500 +I0913 21:10:00.272091 140485033150208 logging_writer.py:48] [15500] global_step=15500, grad_norm=0.5, loss=3.47317 +I0913 21:10:00.280125 140507706811584 submission.py:307] 15500) loss = 3.473, grad_norm = 0.500 +I0913 21:17:29.914254 140485083506432 logging_writer.py:48] [16000] global_step=16000, grad_norm=0.5, loss=3.33265 +I0913 21:17:29.918375 140507706811584 submission.py:307] 16000) loss = 3.333, grad_norm = 0.500 +I0913 21:23:50.400045 140485033150208 logging_writer.py:48] [16500] global_step=16500, grad_norm=0.5, loss=4.16243 +I0913 21:23:50.405690 140507706811584 submission.py:307] 16500) loss = 4.162, grad_norm = 0.500 +I0913 21:28:18.263511 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 21:29:36.719379 140507706811584 spec.py:346] Evaluating on the validation split. +I0913 21:30:42.251025 140507706811584 spec.py:363] Evaluating on the test split. +I0913 21:30:43.209320 140507706811584 submission_runner.py:516] Time since start: 14245.19s, Step: 16925, {'train/accuracy': 0.50466796875, 'train/loss': 3.1147216796875, 'validation/accuracy': 0.4406, 'validation/loss': 3.221945, 'validation/num_examples': 50000, 'test/accuracy': 0.4293, 'test/loss': 2.6210931640625, 'test/num_examples': 10000, 'score': 12935.96284532547, 'total_duration': 14245.187555074692, 'accumulated_submission_time': 12935.96284532547, 'accumulated_eval_time': 1254.5655632019043, 'accumulated_logging_time': 1.678205966949463} +I0913 21:30:43.576874 140488370267904 logging_writer.py:48] [16925] accumulated_eval_time=1254.57, accumulated_logging_time=1.67821, accumulated_submission_time=12936, global_step=16925, preemption_count=0, score=12936, test/accuracy=0.4293, test/loss=2.62109, test/num_examples=10000, total_duration=14245.2, train/accuracy=0.504668, train/loss=3.11472, validation/accuracy=0.4406, validation/loss=3.22194, validation/num_examples=50000 +I0913 21:31:03.137775 140484870616832 logging_writer.py:48] [17000] global_step=17000, grad_norm=0.499999, loss=5.17181 +I0913 21:31:03.140995 140507706811584 submission.py:307] 17000) loss = 5.172, grad_norm = 0.500 +I0913 21:39:55.854358 140488370267904 logging_writer.py:48] [17500] global_step=17500, grad_norm=0.5, loss=2.61979 +I0913 21:39:55.859855 140507706811584 submission.py:307] 17500) loss = 2.620, grad_norm = 0.500 +I0913 21:44:45.126787 140484870616832 logging_writer.py:48] [18000] global_step=18000, grad_norm=0.5, loss=2.66058 +I0913 21:44:45.143280 140507706811584 submission.py:307] 18000) loss = 2.661, grad_norm = 0.500 +I0913 21:52:15.053747 140488370267904 logging_writer.py:48] [18500] global_step=18500, grad_norm=0.5, loss=2.65726 +I0913 21:52:15.057654 140507706811584 submission.py:307] 18500) loss = 2.657, grad_norm = 0.500 +I0913 21:58:38.805033 140484870616832 logging_writer.py:48] [19000] global_step=19000, grad_norm=0.5, loss=2.66722 +I0913 21:58:38.813521 140507706811584 submission.py:307] 19000) loss = 2.667, grad_norm = 0.500 +I0913 22:03:55.819147 140488370267904 logging_writer.py:48] [19500] global_step=19500, grad_norm=0.5, loss=3.87286 +I0913 22:03:55.824563 140507706811584 submission.py:307] 19500) loss = 3.873, grad_norm = 0.500 +I0913 22:12:30.144474 140484870616832 logging_writer.py:48] [20000] global_step=20000, grad_norm=0.5, loss=2.46769 +I0913 22:12:30.178196 140507706811584 submission.py:307] 20000) loss = 2.468, grad_norm = 0.500 +I0913 22:13:38.279891 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 22:14:54.233590 140507706811584 spec.py:346] Evaluating on the validation split. +I0913 22:15:59.923478 140507706811584 spec.py:363] Evaluating on the test split. +I0913 22:16:00.893621 140507706811584 submission_runner.py:516] Time since start: 16962.87s, Step: 20132, {'train/accuracy': 0.51876953125, 'train/loss': 3.346199645996094, 'validation/accuracy': 0.48268, 'validation/loss': 3.011945, 'validation/num_examples': 50000, 'test/accuracy': 0.4636, 'test/loss': 2.4247896484375, 'test/num_examples': 10000, 'score': 15502.118244171143, 'total_duration': 16962.870960712433, 'accumulated_submission_time': 15502.118244171143, 'accumulated_eval_time': 1397.178419828415, 'accumulated_logging_time': 2.4803810119628906} +I0913 22:16:01.229456 140485108684544 logging_writer.py:48] [20132] accumulated_eval_time=1397.18, accumulated_logging_time=2.48038, accumulated_submission_time=15502.1, global_step=20132, preemption_count=0, score=15502.1, test/accuracy=0.4636, test/loss=2.42479, test/num_examples=10000, total_duration=16962.9, train/accuracy=0.51877, train/loss=3.3462, validation/accuracy=0.48268, validation/loss=3.01194, validation/num_examples=50000 +I0913 22:20:37.140638 140484954543872 logging_writer.py:48] [20500] global_step=20500, grad_norm=0.5, loss=2.87706 +I0913 22:20:37.146893 140507706811584 submission.py:307] 20500) loss = 2.877, grad_norm = 0.500 +I0913 22:28:55.434980 140485108684544 logging_writer.py:48] [21000] global_step=21000, grad_norm=0.5, loss=2.42102 +I0913 22:28:55.441992 140507706811584 submission.py:307] 21000) loss = 2.421, grad_norm = 0.500 +I0913 22:35:36.123169 140484954543872 logging_writer.py:48] [21500] global_step=21500, grad_norm=0.499999, loss=5.13988 +I0913 22:35:36.182611 140507706811584 submission.py:307] 21500) loss = 5.140, grad_norm = 0.500 +I0913 22:40:56.928084 140485108684544 logging_writer.py:48] [22000] global_step=22000, grad_norm=0.5, loss=3.01511 +I0913 22:40:56.932225 140507706811584 submission.py:307] 22000) loss = 3.015, grad_norm = 0.500 +I0913 22:50:00.768450 140484954543872 logging_writer.py:48] [22500] global_step=22500, grad_norm=0.5, loss=3.30866 +I0913 22:50:00.772373 140507706811584 submission.py:307] 22500) loss = 3.309, grad_norm = 0.500 +I0913 22:54:20.253077 140485108684544 logging_writer.py:48] [23000] global_step=23000, grad_norm=0.5, loss=2.48931 +I0913 22:54:20.259662 140507706811584 submission.py:307] 23000) loss = 2.489, grad_norm = 0.500 +I0913 22:58:56.840461 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 23:00:18.155852 140507706811584 spec.py:346] Evaluating on the validation split. +I0913 23:01:22.945158 140507706811584 spec.py:363] Evaluating on the test split. +I0913 23:01:23.914093 140507706811584 submission_runner.py:516] Time since start: 19685.89s, Step: 23359, {'train/accuracy': 0.56783203125, 'train/loss': 2.9495895385742186, 'validation/accuracy': 0.50794, 'validation/loss': 3.08117125, 'validation/num_examples': 50000, 'test/accuracy': 0.4881, 'test/loss': 2.2952814453125, 'test/num_examples': 10000, 'score': 18068.6707174778, 'total_duration': 19685.892365455627, 'accumulated_submission_time': 18068.6707174778, 'accumulated_eval_time': 1544.2523880004883, 'accumulated_logging_time': 2.8307547569274902} +I0913 23:01:24.199766 140485091899136 logging_writer.py:48] [23359] accumulated_eval_time=1544.25, accumulated_logging_time=2.83075, accumulated_submission_time=18068.7, global_step=23359, preemption_count=0, score=18068.7, test/accuracy=0.4881, test/loss=2.29528, test/num_examples=10000, total_duration=19685.9, train/accuracy=0.567832, train/loss=2.94959, validation/accuracy=0.50794, validation/loss=3.08117, validation/num_examples=50000 +I0913 23:02:52.771046 140485117077248 logging_writer.py:48] [23500] global_step=23500, grad_norm=0.5, loss=2.67311 +I0913 23:02:52.775172 140507706811584 submission.py:307] 23500) loss = 2.673, grad_norm = 0.500 +I0913 23:09:42.638620 140485091899136 logging_writer.py:48] [24000] global_step=24000, grad_norm=0.499999, loss=4.96016 +I0913 23:09:42.650144 140507706811584 submission.py:307] 24000) loss = 4.960, grad_norm = 0.500 +I0913 23:15:26.066794 140485117077248 logging_writer.py:48] [24500] global_step=24500, grad_norm=0.5, loss=2.55868 +I0913 23:15:26.070953 140507706811584 submission.py:307] 24500) loss = 2.559, grad_norm = 0.500 +I0913 23:24:01.288239 140485091899136 logging_writer.py:48] [25000] global_step=25000, grad_norm=0.5, loss=2.3998 +I0913 23:24:01.292117 140507706811584 submission.py:307] 25000) loss = 2.400, grad_norm = 0.500 +I0913 23:28:17.968963 140485117077248 logging_writer.py:48] [25500] global_step=25500, grad_norm=0.499999, loss=4.08667 +I0913 23:28:17.973561 140507706811584 submission.py:307] 25500) loss = 4.087, grad_norm = 0.500 +I0913 23:35:32.761565 140485091899136 logging_writer.py:48] [26000] global_step=26000, grad_norm=0.5, loss=2.1701 +I0913 23:35:32.765537 140507706811584 submission.py:307] 26000) loss = 2.170, grad_norm = 0.500 +I0913 23:42:04.834136 140485117077248 logging_writer.py:48] [26500] global_step=26500, grad_norm=0.5, loss=2.32553 +I0913 23:42:04.839908 140507706811584 submission.py:307] 26500) loss = 2.326, grad_norm = 0.500 +I0913 23:44:19.861307 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 23:45:38.434881 140507706811584 spec.py:346] Evaluating on the validation split. +I0913 23:46:43.707140 140507706811584 spec.py:363] Evaluating on the test split. +I0913 23:46:44.583355 140507706811584 submission_runner.py:516] Time since start: 22406.56s, Step: 26742, {'train/accuracy': 0.60248046875, 'train/loss': 2.7548953247070314, 'validation/accuracy': 0.52264, 'validation/loss': 3.144949375, 'validation/num_examples': 50000, 'test/accuracy': 0.5129, 'test/loss': 2.1553640625, 'test/num_examples': 10000, 'score': 20635.430467367172, 'total_duration': 22406.56116294861, 'accumulated_submission_time': 20635.430467367172, 'accumulated_eval_time': 1688.974037885666, 'accumulated_logging_time': 3.12606143951416} +I0913 23:46:44.929276 140485075113728 logging_writer.py:48] [26742] accumulated_eval_time=1688.97, accumulated_logging_time=3.12606, accumulated_submission_time=20635.4, global_step=26742, preemption_count=0, score=20635.4, test/accuracy=0.5129, test/loss=2.15536, test/num_examples=10000, total_duration=22406.6, train/accuracy=0.60248, train/loss=2.7549, validation/accuracy=0.52264, validation/loss=3.14495, validation/num_examples=50000 +I0913 23:50:02.635313 140485066721024 logging_writer.py:48] [27000] global_step=27000, grad_norm=0.499999, loss=4.44539 +I0913 23:50:02.640600 140507706811584 submission.py:307] 27000) loss = 4.445, grad_norm = 0.500 +I0913 23:59:07.829545 140485075113728 logging_writer.py:48] [27500] global_step=27500, grad_norm=0.5, loss=2.13355 +I0913 23:59:07.833452 140507706811584 submission.py:307] 27500) loss = 2.134, grad_norm = 0.500 +I0914 00:03:33.971772 140485066721024 logging_writer.py:48] [28000] global_step=28000, grad_norm=0.5, loss=2.00063 +I0914 00:03:33.977619 140507706811584 submission.py:307] 28000) loss = 2.001, grad_norm = 0.500 +I0914 00:10:56.314716 140485075113728 logging_writer.py:48] [28500] global_step=28500, grad_norm=0.5, loss=2.569 +I0914 00:10:56.320175 140507706811584 submission.py:307] 28500) loss = 2.569, grad_norm = 0.500 +I0914 00:17:35.724358 140485066721024 logging_writer.py:48] [29000] global_step=29000, grad_norm=0.5, loss=4.73446 +I0914 00:17:35.730014 140507706811584 submission.py:307] 29000) loss = 4.734, grad_norm = 0.500 +I0914 00:22:49.242191 140485075113728 logging_writer.py:48] [29500] global_step=29500, grad_norm=0.5, loss=2.76517 +I0914 00:22:49.246443 140507706811584 submission.py:307] 29500) loss = 2.765, grad_norm = 0.500 +I0914 00:29:40.722799 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 00:31:12.936761 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 00:32:17.940257 140507706811584 spec.py:363] Evaluating on the test split. +I0914 00:32:18.807833 140507706811584 submission_runner.py:516] Time since start: 25140.79s, Step: 29900, {'train/accuracy': 0.61646484375, 'train/loss': 2.706271057128906, 'validation/accuracy': 0.53204, 'validation/loss': 2.973585625, 'validation/num_examples': 50000, 'test/accuracy': 0.5292, 'test/loss': 2.0948826171875, 'test/num_examples': 10000, 'score': 23202.31759428978, 'total_duration': 25140.786123991013, 'accumulated_submission_time': 23202.31759428978, 'accumulated_eval_time': 1847.0592744350433, 'accumulated_logging_time': 3.481067657470703} +I0914 00:32:19.086487 140485041542912 logging_writer.py:48] [29900] accumulated_eval_time=1847.06, accumulated_logging_time=3.48107, accumulated_submission_time=23202.3, global_step=29900, preemption_count=0, score=23202.3, test/accuracy=0.5292, test/loss=2.09488, test/num_examples=10000, total_duration=25140.8, train/accuracy=0.616465, train/loss=2.70627, validation/accuracy=0.53204, validation/loss=2.97359, validation/num_examples=50000 +I0914 00:33:02.780231 140484991186688 logging_writer.py:48] [30000] global_step=30000, grad_norm=0.5, loss=3.25955 +I0914 00:33:02.786874 140507706811584 submission.py:307] 30000) loss = 3.260, grad_norm = 0.500 +I0914 00:39:15.752455 140485041542912 logging_writer.py:48] [30500] global_step=30500, grad_norm=0.5, loss=2.13716 +I0914 00:39:15.756552 140507706811584 submission.py:307] 30500) loss = 2.137, grad_norm = 0.500 +I0914 00:47:05.491037 140484991186688 logging_writer.py:48] [31000] global_step=31000, grad_norm=0.5, loss=4.00406 +I0914 00:47:05.494916 140507706811584 submission.py:307] 31000) loss = 4.004, grad_norm = 0.500 +I0914 00:53:33.663945 140485041542912 logging_writer.py:48] [31500] global_step=31500, grad_norm=0.5, loss=2.11477 +I0914 00:53:33.677060 140507706811584 submission.py:307] 31500) loss = 2.115, grad_norm = 0.500 +I0914 00:59:01.206918 140484991186688 logging_writer.py:48] [32000] global_step=32000, grad_norm=0.5, loss=2.06724 +I0914 00:59:01.220621 140507706811584 submission.py:307] 32000) loss = 2.067, grad_norm = 0.500 +I0914 01:07:58.320337 140485041542912 logging_writer.py:48] [32500] global_step=32500, grad_norm=0.5, loss=2.63493 +I0914 01:07:58.325182 140507706811584 submission.py:307] 32500) loss = 2.635, grad_norm = 0.500 +I0914 01:12:09.746296 140484991186688 logging_writer.py:48] [33000] global_step=33000, grad_norm=0.5, loss=2.06046 +I0914 01:12:09.753535 140507706811584 submission.py:307] 33000) loss = 2.060, grad_norm = 0.500 +I0914 01:15:15.098690 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 01:16:33.862797 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 01:17:39.232221 140507706811584 spec.py:363] Evaluating on the test split. +I0914 01:17:40.160693 140507706811584 submission_runner.py:516] Time since start: 27862.14s, Step: 33252, {'train/accuracy': 0.6110546875, 'train/loss': 2.8116305541992186, 'validation/accuracy': 0.55746, 'validation/loss': 2.6731121875, 'validation/num_examples': 50000, 'test/accuracy': 0.543, 'test/loss': 2.029508984375, 'test/num_examples': 10000, 'score': 25768.509074926376, 'total_duration': 27862.138606786728, 'accumulated_submission_time': 25768.509074926376, 'accumulated_eval_time': 1992.1209709644318, 'accumulated_logging_time': 3.7691237926483154} +I0914 01:17:40.498219 140485033150208 logging_writer.py:48] [33252] accumulated_eval_time=1992.12, accumulated_logging_time=3.76912, accumulated_submission_time=25768.5, global_step=33252, preemption_count=0, score=25768.5, test/accuracy=0.543, test/loss=2.02951, test/num_examples=10000, total_duration=27862.1, train/accuracy=0.611055, train/loss=2.81163, validation/accuracy=0.55746, validation/loss=2.67311, validation/num_examples=50000 +I0914 01:21:01.890878 140488370267904 logging_writer.py:48] [33500] global_step=33500, grad_norm=0.499999, loss=2.11584 +I0914 01:21:01.895834 140507706811584 submission.py:307] 33500) loss = 2.116, grad_norm = 0.500 +I0914 01:27:57.804274 140485033150208 logging_writer.py:48] [34000] global_step=34000, grad_norm=0.499999, loss=3.32006 +I0914 01:27:57.808295 140507706811584 submission.py:307] 34000) loss = 3.320, grad_norm = 0.500 +I0914 01:33:33.676493 140488370267904 logging_writer.py:48] [34500] global_step=34500, grad_norm=0.5, loss=2.8897 +I0914 01:33:33.685265 140507706811584 submission.py:307] 34500) loss = 2.890, grad_norm = 0.500 +I0914 01:42:16.479487 140485033150208 logging_writer.py:48] [35000] global_step=35000, grad_norm=0.5, loss=4.14507 +I0914 01:42:16.483484 140507706811584 submission.py:307] 35000) loss = 4.145, grad_norm = 0.500 +I0914 01:46:31.913008 140488370267904 logging_writer.py:48] [35500] global_step=35500, grad_norm=0.5, loss=1.9228 +I0914 01:46:31.936459 140507706811584 submission.py:307] 35500) loss = 1.923, grad_norm = 0.500 +I0914 01:53:39.483709 140485033150208 logging_writer.py:48] [36000] global_step=36000, grad_norm=0.499999, loss=3.07942 +I0914 01:53:39.489565 140507706811584 submission.py:307] 36000) loss = 3.079, grad_norm = 0.500 +I0914 02:00:11.971415 140488370267904 logging_writer.py:48] [36500] global_step=36500, grad_norm=0.5, loss=1.85386 +I0914 02:00:11.977805 140507706811584 submission.py:307] 36500) loss = 1.854, grad_norm = 0.500 +I0914 02:00:36.132511 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 02:01:46.522519 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 02:02:52.446750 140507706811584 spec.py:363] Evaluating on the test split. +I0914 02:02:53.497516 140507706811584 submission_runner.py:516] Time since start: 30575.48s, Step: 36543, {'train/accuracy': 0.66943359375, 'train/loss': 2.203136749267578, 'validation/accuracy': 0.5562, 'validation/loss': 2.953885625, 'validation/num_examples': 50000, 'test/accuracy': 0.5498, 'test/loss': 1.9853494140625, 'test/num_examples': 10000, 'score': 28333.736892700195, 'total_duration': 30575.475831747055, 'accumulated_submission_time': 28333.736892700195, 'accumulated_eval_time': 2129.4862949848175, 'accumulated_logging_time': 4.115705490112305} +I0914 02:02:53.898321 140485049935616 logging_writer.py:48] [36543] accumulated_eval_time=2129.49, accumulated_logging_time=4.11571, accumulated_submission_time=28333.7, global_step=36543, preemption_count=0, score=28333.7, test/accuracy=0.5498, test/loss=1.98535, test/num_examples=10000, total_duration=30575.5, train/accuracy=0.669434, train/loss=2.20314, validation/accuracy=0.5562, validation/loss=2.95389, validation/num_examples=50000 +I0914 02:09:28.801284 140485016364800 logging_writer.py:48] [37000] global_step=37000, grad_norm=0.5, loss=1.97306 +I0914 02:09:28.805123 140507706811584 submission.py:307] 37000) loss = 1.973, grad_norm = 0.500 +I0914 02:18:51.775966 140485049935616 logging_writer.py:48] [37500] global_step=37500, grad_norm=0.5, loss=3.02389 +I0914 02:18:51.779729 140507706811584 submission.py:307] 37500) loss = 3.024, grad_norm = 0.500 +I0914 02:23:23.341226 140485016364800 logging_writer.py:48] [38000] global_step=38000, grad_norm=0.499999, loss=2.25712 +I0914 02:23:23.345234 140507706811584 submission.py:307] 38000) loss = 2.257, grad_norm = 0.500 +I0914 02:30:47.151433 140485049935616 logging_writer.py:48] [38500] global_step=38500, grad_norm=0.5, loss=2.31605 +I0914 02:30:47.158165 140507706811584 submission.py:307] 38500) loss = 2.316, grad_norm = 0.500 +I0914 02:37:30.081587 140485016364800 logging_writer.py:48] [39000] global_step=39000, grad_norm=0.499999, loss=2.27127 +I0914 02:37:30.087105 140507706811584 submission.py:307] 39000) loss = 2.271, grad_norm = 0.500 +I0914 02:42:40.901678 140485049935616 logging_writer.py:48] [39500] global_step=39500, grad_norm=0.5, loss=1.81748 +I0914 02:42:40.907708 140507706811584 submission.py:307] 39500) loss = 1.817, grad_norm = 0.500 +I0914 02:45:50.913775 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 02:47:18.101367 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 02:48:24.186908 140507706811584 spec.py:363] Evaluating on the test split. +I0914 02:48:25.109111 140507706811584 submission_runner.py:516] Time since start: 33307.09s, Step: 39689, {'train/accuracy': 0.64908203125, 'train/loss': 2.6580075073242186, 'validation/accuracy': 0.5803, 'validation/loss': 2.6447209375, 'validation/num_examples': 50000, 'test/accuracy': 0.5596, 'test/loss': 1.9409359375, 'test/num_examples': 10000, 'score': 30901.944776058197, 'total_duration': 33307.08735227585, 'accumulated_submission_time': 30901.944776058197, 'accumulated_eval_time': 2283.681713104248, 'accumulated_logging_time': 4.5361340045928955} +I0914 02:48:25.417894 140485075113728 logging_writer.py:48] [39689] accumulated_eval_time=2283.68, accumulated_logging_time=4.53613, accumulated_submission_time=30901.9, global_step=39689, preemption_count=0, score=30901.9, test/accuracy=0.5596, test/loss=1.94094, test/num_examples=10000, total_duration=33307.1, train/accuracy=0.649082, train/loss=2.65801, validation/accuracy=0.5803, validation/loss=2.64472, validation/num_examples=50000 +I0914 02:53:10.516127 140488490288896 logging_writer.py:48] [40000] global_step=40000, grad_norm=0.5, loss=4.50804 +I0914 02:53:10.519899 140507706811584 submission.py:307] 40000) loss = 4.508, grad_norm = 0.500 +I0914 02:58:17.558240 140485075113728 logging_writer.py:48] [40500] global_step=40500, grad_norm=0.5, loss=3.13663 +I0914 02:58:17.562234 140507706811584 submission.py:307] 40500) loss = 3.137, grad_norm = 0.500 +I0914 03:05:38.639311 140488490288896 logging_writer.py:48] [41000] global_step=41000, grad_norm=0.5, loss=1.83513 +I0914 03:05:38.644808 140507706811584 submission.py:307] 41000) loss = 1.835, grad_norm = 0.500 +I0914 03:12:03.132835 140485075113728 logging_writer.py:48] [41500] global_step=41500, grad_norm=0.5, loss=1.76587 +I0914 03:12:03.157637 140507706811584 submission.py:307] 41500) loss = 1.766, grad_norm = 0.500 +I0914 03:17:20.494289 140488490288896 logging_writer.py:48] [42000] global_step=42000, grad_norm=0.499999, loss=2.41458 +I0914 03:17:20.498267 140507706811584 submission.py:307] 42000) loss = 2.415, grad_norm = 0.500 +I0914 03:26:03.696588 140485075113728 logging_writer.py:48] [42500] global_step=42500, grad_norm=0.5, loss=1.82296 +I0914 03:26:03.700551 140507706811584 submission.py:307] 42500) loss = 1.823, grad_norm = 0.500 +I0914 03:30:27.678592 140488490288896 logging_writer.py:48] [43000] global_step=43000, grad_norm=0.5, loss=3.91981 +I0914 03:30:27.683766 140507706811584 submission.py:307] 43000) loss = 3.920, grad_norm = 0.500 +I0914 03:31:22.689779 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 03:32:38.274614 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 03:33:43.724537 140507706811584 spec.py:363] Evaluating on the test split. +I0914 03:33:44.587241 140507706811584 submission_runner.py:516] Time since start: 36026.56s, Step: 43077, {'train/accuracy': 0.65333984375, 'train/loss': 2.6797079467773437, 'validation/accuracy': 0.57702, 'validation/loss': 2.819873125, 'validation/num_examples': 50000, 'test/accuracy': 0.5687, 'test/loss': 1.913438671875, 'test/num_examples': 10000, 'score': 33470.53742003441, 'total_duration': 36026.5645134449, 'accumulated_submission_time': 33470.53742003441, 'accumulated_eval_time': 2425.578112602234, 'accumulated_logging_time': 4.859193801879883} +I0914 03:33:44.966559 140485108684544 logging_writer.py:48] [43077] accumulated_eval_time=2425.58, accumulated_logging_time=4.85919, accumulated_submission_time=33470.5, global_step=43077, preemption_count=0, score=33470.5, test/accuracy=0.5687, test/loss=1.91344, test/num_examples=10000, total_duration=36026.6, train/accuracy=0.65334, train/loss=2.67971, validation/accuracy=0.57702, validation/loss=2.81987, validation/num_examples=50000 +I0914 03:40:05.032321 140485091899136 logging_writer.py:48] [43500] global_step=43500, grad_norm=0.5, loss=1.87477 +I0914 03:40:05.036336 140507706811584 submission.py:307] 43500) loss = 1.875, grad_norm = 0.500 +I0914 03:46:55.619300 140485108684544 logging_writer.py:48] [44000] global_step=44000, grad_norm=0.5, loss=4.37599 +I0914 03:46:55.624386 140507706811584 submission.py:307] 44000) loss = 4.376, grad_norm = 0.500 +I0914 03:52:21.485638 140485091899136 logging_writer.py:48] [44500] global_step=44500, grad_norm=0.5, loss=2.25207 +I0914 03:52:21.489541 140507706811584 submission.py:307] 44500) loss = 2.252, grad_norm = 0.500 +I0914 04:01:12.349792 140485108684544 logging_writer.py:48] [45000] global_step=45000, grad_norm=0.5, loss=2.7875 +I0914 04:01:12.355042 140507706811584 submission.py:307] 45000) loss = 2.787, grad_norm = 0.500 +I0914 04:05:35.559515 140485091899136 logging_writer.py:48] [45500] global_step=45500, grad_norm=0.5, loss=2.27176 +I0914 04:05:35.565551 140507706811584 submission.py:307] 45500) loss = 2.272, grad_norm = 0.500 +I0914 04:12:39.584310 140485108684544 logging_writer.py:48] [46000] global_step=46000, grad_norm=0.5, loss=2.47579 +I0914 04:12:39.588811 140507706811584 submission.py:307] 46000) loss = 2.476, grad_norm = 0.500 +I0914 04:16:48.312987 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 04:17:53.422279 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 04:18:58.653696 140507706811584 spec.py:363] Evaluating on the test split. +I0914 04:18:59.614878 140507706811584 submission_runner.py:516] Time since start: 38741.59s, Step: 46226, {'train/accuracy': 0.66466796875, 'train/loss': 2.5644097900390626, 'validation/accuracy': 0.58578, 'validation/loss': 2.784920625, 'validation/num_examples': 50000, 'test/accuracy': 0.5766, 'test/loss': 1.855303515625, 'test/num_examples': 10000, 'score': 36045.62726044655, 'total_duration': 38741.59304046631, 'accumulated_submission_time': 36045.62726044655, 'accumulated_eval_time': 2556.88010764122, 'accumulated_logging_time': 5.248838663101196} +I0914 04:18:59.988147 140484991186688 logging_writer.py:48] [46226] accumulated_eval_time=2556.88, accumulated_logging_time=5.24884, accumulated_submission_time=36045.6, global_step=46226, preemption_count=0, score=36045.6, test/accuracy=0.5766, test/loss=1.8553, test/num_examples=10000, total_duration=38741.6, train/accuracy=0.664668, train/loss=2.56441, validation/accuracy=0.58578, validation/loss=2.78492, validation/num_examples=50000 +I0914 04:22:13.737420 140485058328320 logging_writer.py:48] [46500] global_step=46500, grad_norm=0.5, loss=1.68741 +I0914 04:22:13.741500 140507706811584 submission.py:307] 46500) loss = 1.687, grad_norm = 0.500 +I0914 04:29:06.678132 140484991186688 logging_writer.py:48] [47000] global_step=47000, grad_norm=0.5, loss=1.82595 +I0914 04:29:06.684429 140507706811584 submission.py:307] 47000) loss = 1.826, grad_norm = 0.500 +I0914 04:38:03.954364 140485058328320 logging_writer.py:48] [47500] global_step=47500, grad_norm=0.5, loss=1.64877 +I0914 04:38:03.958458 140507706811584 submission.py:307] 47500) loss = 1.649, grad_norm = 0.500 +I0914 04:42:22.587289 140484991186688 logging_writer.py:48] [48000] global_step=48000, grad_norm=0.5, loss=2.53269 +I0914 04:42:22.591618 140507706811584 submission.py:307] 48000) loss = 2.533, grad_norm = 0.500 +I0914 04:49:57.223507 140485058328320 logging_writer.py:48] [48500] global_step=48500, grad_norm=0.5, loss=3.77043 +I0914 04:49:57.228631 140507706811584 submission.py:307] 48500) loss = 3.770, grad_norm = 0.500 +I0914 04:56:53.285837 140484991186688 logging_writer.py:48] [49000] global_step=49000, grad_norm=0.5, loss=1.95992 +I0914 04:56:53.290967 140507706811584 submission.py:307] 49000) loss = 1.960, grad_norm = 0.500 +I0914 05:01:55.450227 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 05:03:16.654059 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 05:04:22.090979 140507706811584 spec.py:363] Evaluating on the test split. +I0914 05:04:23.035267 140507706811584 submission_runner.py:516] Time since start: 41465.01s, Step: 49479, {'train/accuracy': 0.6749609375, 'train/loss': 2.5443516540527344, 'validation/accuracy': 0.58796, 'validation/loss': 2.8763503125, 'validation/num_examples': 50000, 'test/accuracy': 0.5787, 'test/loss': 1.857298828125, 'test/num_examples': 10000, 'score': 38612.53212809563, 'total_duration': 41465.013504981995, 'accumulated_submission_time': 38612.53212809563, 'accumulated_eval_time': 2704.4651052951813, 'accumulated_logging_time': 5.631556510925293} +I0914 05:04:23.319354 140488498681600 logging_writer.py:48] [49479] accumulated_eval_time=2704.47, accumulated_logging_time=5.63156, accumulated_submission_time=38612.5, global_step=49479, preemption_count=0, score=38612.5, test/accuracy=0.5787, test/loss=1.8573, test/num_examples=10000, total_duration=41465, train/accuracy=0.674961, train/loss=2.54435, validation/accuracy=0.58796, validation/loss=2.87635, validation/num_examples=50000 +I0914 05:04:29.623547 140485100291840 logging_writer.py:48] [49500] global_step=49500, grad_norm=0.499999, loss=3.12152 +I0914 05:04:29.627777 140507706811584 submission.py:307] 49500) loss = 3.122, grad_norm = 0.500 +I0914 05:12:52.600947 140488498681600 logging_writer.py:48] [50000] global_step=50000, grad_norm=0.5, loss=2.27806 +I0914 05:12:52.604777 140507706811584 submission.py:307] 50000) loss = 2.278, grad_norm = 0.500 +I0914 05:17:47.753681 140485100291840 logging_writer.py:48] [50500] global_step=50500, grad_norm=0.5, loss=1.60937 +I0914 05:17:47.760026 140507706811584 submission.py:307] 50500) loss = 1.609, grad_norm = 0.500 +I0914 05:25:05.727649 140488498681600 logging_writer.py:48] [51000] global_step=51000, grad_norm=0.5, loss=1.73819 +I0914 05:25:05.731651 140507706811584 submission.py:307] 51000) loss = 1.738, grad_norm = 0.500 +I0914 05:31:47.906938 140485100291840 logging_writer.py:48] [51500] global_step=51500, grad_norm=0.5, loss=1.66249 +I0914 05:31:47.913219 140507706811584 submission.py:307] 51500) loss = 1.662, grad_norm = 0.500 +I0914 05:37:04.797087 140488498681600 logging_writer.py:48] [52000] global_step=52000, grad_norm=0.5, loss=2.35659 +I0914 05:37:04.801399 140507706811584 submission.py:307] 52000) loss = 2.357, grad_norm = 0.500 +I0914 05:45:46.722691 140485100291840 logging_writer.py:48] [52500] global_step=52500, grad_norm=0.5, loss=1.84133 +I0914 05:45:46.726847 140507706811584 submission.py:307] 52500) loss = 1.841, grad_norm = 0.500 +I0914 05:47:20.412406 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 05:48:33.505995 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 05:49:38.927797 140507706811584 spec.py:363] Evaluating on the test split. +I0914 05:49:39.940881 140507706811584 submission_runner.py:516] Time since start: 44181.92s, Step: 52682, {'train/accuracy': 0.67267578125, 'train/loss': 2.7012420654296876, 'validation/accuracy': 0.59154, 'validation/loss': 2.840863125, 'validation/num_examples': 50000, 'test/accuracy': 0.5899, 'test/loss': 1.833320703125, 'test/num_examples': 10000, 'score': 41181.36528348923, 'total_duration': 44181.918159246445, 'accumulated_submission_time': 41181.36528348923, 'accumulated_eval_time': 2843.9927847385406, 'accumulated_logging_time': 5.925373315811157} +I0914 05:49:40.272315 140485033150208 logging_writer.py:48] [52682] accumulated_eval_time=2843.99, accumulated_logging_time=5.92537, accumulated_submission_time=41181.4, global_step=52682, preemption_count=0, score=41181.4, test/accuracy=0.5899, test/loss=1.83332, test/num_examples=10000, total_duration=44181.9, train/accuracy=0.672676, train/loss=2.70124, validation/accuracy=0.59154, validation/loss=2.84086, validation/num_examples=50000 +I0914 05:53:47.185451 140488370267904 logging_writer.py:48] [53000] global_step=53000, grad_norm=0.5, loss=3.26189 +I0914 05:53:47.189259 140507706811584 submission.py:307] 53000) loss = 3.262, grad_norm = 0.500 +I0914 06:02:14.760335 140485033150208 logging_writer.py:48] [53500] global_step=53500, grad_norm=0.5, loss=1.59751 +I0914 06:02:14.769674 140507706811584 submission.py:307] 53500) loss = 1.598, grad_norm = 0.500 +I0914 06:09:56.379558 140488370267904 logging_writer.py:48] [54000] global_step=54000, grad_norm=0.5, loss=3.87363 +I0914 06:09:56.385813 140507706811584 submission.py:307] 54000) loss = 3.874, grad_norm = 0.500 +I0914 06:15:00.493240 140485033150208 logging_writer.py:48] [54500] global_step=54500, grad_norm=0.5, loss=3.57067 +I0914 06:15:00.498908 140507706811584 submission.py:307] 54500) loss = 3.571, grad_norm = 0.500 +I0914 06:24:18.137655 140488370267904 logging_writer.py:48] [55000] global_step=55000, grad_norm=0.5, loss=1.71288 +I0914 06:24:18.142778 140507706811584 submission.py:307] 55000) loss = 1.713, grad_norm = 0.500 +I0914 06:28:41.249061 140485033150208 logging_writer.py:48] [55500] global_step=55500, grad_norm=0.5, loss=1.50716 +I0914 06:28:41.253380 140507706811584 submission.py:307] 55500) loss = 1.507, grad_norm = 0.500 +I0914 06:32:37.689980 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 06:33:57.464093 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 06:35:02.702989 140507706811584 spec.py:363] Evaluating on the test split. +I0914 06:35:03.591237 140507706811584 submission_runner.py:516] Time since start: 46905.57s, Step: 55813, {'train/accuracy': 0.6837109375, 'train/loss': 2.5283877563476564, 'validation/accuracy': 0.61526, 'validation/loss': 2.4480253125, 'validation/num_examples': 50000, 'test/accuracy': 0.5934, 'test/loss': 1.80217890625, 'test/num_examples': 10000, 'score': 43747.88288974762, 'total_duration': 46905.568497657776, 'accumulated_submission_time': 43747.88288974762, 'accumulated_eval_time': 2989.893040895462, 'accumulated_logging_time': 6.267249822616577} +I0914 06:35:03.949196 140484999579392 logging_writer.py:48] [55813] accumulated_eval_time=2989.89, accumulated_logging_time=6.26725, accumulated_submission_time=43747.9, global_step=55813, preemption_count=0, score=43747.9, test/accuracy=0.5934, test/loss=1.80218, test/num_examples=10000, total_duration=46905.6, train/accuracy=0.683711, train/loss=2.52839, validation/accuracy=0.61526, validation/loss=2.44803, validation/num_examples=50000 +I0914 06:37:23.211224 140485024757504 logging_writer.py:48] [56000] global_step=56000, grad_norm=0.5, loss=3.44324 +I0914 06:37:23.216277 140507706811584 submission.py:307] 56000) loss = 3.443, grad_norm = 0.500 +I0914 06:44:40.346359 140484999579392 logging_writer.py:48] [56500] global_step=56500, grad_norm=0.5, loss=1.51943 +I0914 06:44:40.351042 140507706811584 submission.py:307] 56500) loss = 1.519, grad_norm = 0.500 +I0914 06:50:15.514361 140485024757504 logging_writer.py:48] [57000] global_step=57000, grad_norm=0.5, loss=1.61804 +I0914 06:50:15.519288 140507706811584 submission.py:307] 57000) loss = 1.618, grad_norm = 0.500 +I0914 06:59:05.340597 140484999579392 logging_writer.py:48] [57500] global_step=57500, grad_norm=0.5, loss=2.91592 +I0914 06:59:05.346580 140507706811584 submission.py:307] 57500) loss = 2.916, grad_norm = 0.500 +I0914 07:03:20.707928 140485024757504 logging_writer.py:48] [58000] global_step=58000, grad_norm=0.5, loss=1.61531 +I0914 07:03:20.713773 140507706811584 submission.py:307] 58000) loss = 1.615, grad_norm = 0.500 +I0914 07:10:30.319145 140484999579392 logging_writer.py:48] [58500] global_step=58500, grad_norm=0.5, loss=1.89493 +I0914 07:10:30.323050 140507706811584 submission.py:307] 58500) loss = 1.895, grad_norm = 0.500 +I0914 07:17:13.514521 140485024757504 logging_writer.py:48] [59000] global_step=59000, grad_norm=0.5, loss=1.68041 +I0914 07:17:13.522423 140507706811584 submission.py:307] 59000) loss = 1.680, grad_norm = 0.500 +I0914 07:17:59.309470 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 07:19:11.749188 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 07:20:17.106508 140507706811584 spec.py:363] Evaluating on the test split. +I0914 07:20:18.066277 140507706811584 submission_runner.py:516] Time since start: 49620.04s, Step: 59086, {'train/accuracy': 0.66517578125, 'train/loss': 2.8801678466796874, 'validation/accuracy': 0.61194, 'validation/loss': 2.4760378125, 'validation/num_examples': 50000, 'test/accuracy': 0.5995, 'test/loss': 1.7874669921875, 'test/num_examples': 10000, 'score': 46314.69552898407, 'total_duration': 49620.04446363449, 'accumulated_submission_time': 46314.69552898407, 'accumulated_eval_time': 3128.6497945785522, 'accumulated_logging_time': 6.634281158447266} +I0914 07:20:18.401620 140484991186688 logging_writer.py:48] [59086] accumulated_eval_time=3128.65, accumulated_logging_time=6.63428, accumulated_submission_time=46314.7, global_step=59086, preemption_count=0, score=46314.7, test/accuracy=0.5995, test/loss=1.78747, test/num_examples=10000, total_duration=49620, train/accuracy=0.665176, train/loss=2.88017, validation/accuracy=0.61194, validation/loss=2.47604, validation/num_examples=50000 +I0914 07:26:13.592047 140485016364800 logging_writer.py:48] [59500] global_step=59500, grad_norm=0.5, loss=1.69781 +I0914 07:26:13.598051 140507706811584 submission.py:307] 59500) loss = 1.698, grad_norm = 0.500 +I0914 07:35:29.575629 140484991186688 logging_writer.py:48] [60000] global_step=60000, grad_norm=0.5, loss=1.56784 +I0914 07:35:29.581617 140507706811584 submission.py:307] 60000) loss = 1.568, grad_norm = 0.500 +I0914 07:40:09.461784 140485016364800 logging_writer.py:48] [60500] global_step=60500, grad_norm=0.5, loss=2.55564 +I0914 07:40:09.466907 140507706811584 submission.py:307] 60500) loss = 2.556, grad_norm = 0.500 +I0914 07:47:24.816457 140484991186688 logging_writer.py:48] [61000] global_step=61000, grad_norm=0.5, loss=1.56961 +I0914 07:47:24.820574 140507706811584 submission.py:307] 61000) loss = 1.570, grad_norm = 0.500 +I0914 07:54:20.187923 140485016364800 logging_writer.py:48] [61500] global_step=61500, grad_norm=0.5, loss=1.91171 +I0914 07:54:20.194314 140507706811584 submission.py:307] 61500) loss = 1.912, grad_norm = 0.500 +I0914 07:59:28.376026 140484991186688 logging_writer.py:48] [62000] global_step=62000, grad_norm=0.5, loss=2.23285 +I0914 07:59:28.381062 140507706811584 submission.py:307] 62000) loss = 2.233, grad_norm = 0.500 +I0914 08:03:13.700515 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 08:04:44.007446 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 08:05:49.413023 140507706811584 spec.py:363] Evaluating on the test split. +I0914 08:05:50.373511 140507706811584 submission_runner.py:516] Time since start: 52352.35s, Step: 62233, {'train/accuracy': 0.6954296875, 'train/loss': 2.4459095764160157, 'validation/accuracy': 0.6085, 'validation/loss': 2.61531984375, 'validation/num_examples': 50000, 'test/accuracy': 0.5972, 'test/loss': 1.782680859375, 'test/num_examples': 10000, 'score': 48881.777411699295, 'total_duration': 52352.351593256, 'accumulated_submission_time': 48881.777411699295, 'accumulated_eval_time': 3285.323085308075, 'accumulated_logging_time': 6.9809653759002686} +I0914 08:05:50.656653 140484999579392 logging_writer.py:48] [62233] accumulated_eval_time=3285.32, accumulated_logging_time=6.98097, accumulated_submission_time=48881.8, global_step=62233, preemption_count=0, score=48881.8, test/accuracy=0.5972, test/loss=1.78268, test/num_examples=10000, total_duration=52352.4, train/accuracy=0.69543, train/loss=2.44591, validation/accuracy=0.6085, validation/loss=2.61532, validation/num_examples=50000 +I0914 08:09:48.174925 140485007972096 logging_writer.py:48] [62500] global_step=62500, grad_norm=0.5, loss=1.61381 +I0914 08:09:48.178829 140507706811584 submission.py:307] 62500) loss = 1.614, grad_norm = 0.500 +I0914 08:15:01.097808 140484999579392 logging_writer.py:48] [63000] global_step=63000, grad_norm=0.5, loss=3.56985 +I0914 08:15:01.102690 140507706811584 submission.py:307] 63000) loss = 3.570, grad_norm = 0.500 +I0914 08:22:16.651680 140485007972096 logging_writer.py:48] [63500] global_step=63500, grad_norm=0.5, loss=2.10526 +I0914 08:22:16.656977 140507706811584 submission.py:307] 63500) loss = 2.105, grad_norm = 0.500 +I0914 08:28:54.200844 140484999579392 logging_writer.py:48] [64000] global_step=64000, grad_norm=0.5, loss=1.45231 +I0914 08:28:54.206628 140507706811584 submission.py:307] 64000) loss = 1.452, grad_norm = 0.500 +I0914 08:34:02.588206 140485007972096 logging_writer.py:48] [64500] global_step=64500, grad_norm=0.5, loss=1.62334 +I0914 08:34:02.594134 140507706811584 submission.py:307] 64500) loss = 1.623, grad_norm = 0.500 +I0914 08:42:46.438277 140484999579392 logging_writer.py:48] [65000] global_step=65000, grad_norm=0.5, loss=1.56603 +I0914 08:42:46.442201 140507706811584 submission.py:307] 65000) loss = 1.566, grad_norm = 0.500 +I0914 08:47:10.432979 140485007972096 logging_writer.py:48] [65500] global_step=65500, grad_norm=0.5, loss=1.57531 +I0914 08:47:10.440686 140507706811584 submission.py:307] 65500) loss = 1.575, grad_norm = 0.500 +I0914 08:48:46.463450 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 08:50:02.911538 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 08:51:07.847461 140507706811584 spec.py:363] Evaluating on the test split. +I0914 08:51:08.719306 140507706811584 submission_runner.py:516] Time since start: 55070.70s, Step: 65639, {'train/accuracy': 0.693125, 'train/loss': 2.52487060546875, 'validation/accuracy': 0.61096, 'validation/loss': 2.59159, 'validation/num_examples': 50000, 'test/accuracy': 0.6034, 'test/loss': 1.7580052734375, 'test/num_examples': 10000, 'score': 51448.57416796684, 'total_duration': 55070.697471380234, 'accumulated_submission_time': 51448.57416796684, 'accumulated_eval_time': 3427.5788867473602, 'accumulated_logging_time': 7.273256301879883} +I0914 08:51:08.993800 140484828653312 logging_writer.py:48] [65639] accumulated_eval_time=3427.58, accumulated_logging_time=7.27326, accumulated_submission_time=51448.6, global_step=65639, preemption_count=0, score=51448.6, test/accuracy=0.6034, test/loss=1.75801, test/num_examples=10000, total_duration=55070.7, train/accuracy=0.693125, train/loss=2.52487, validation/accuracy=0.61096, validation/loss=2.59159, validation/num_examples=50000 +I0914 08:56:27.439255 140485041542912 logging_writer.py:48] [66000] global_step=66000, grad_norm=0.5, loss=1.71491 +I0914 08:56:27.443756 140507706811584 submission.py:307] 66000) loss = 1.715, grad_norm = 0.500 +I0914 09:03:32.373220 140484828653312 logging_writer.py:48] [66500] global_step=66500, grad_norm=0.5, loss=2.57043 +I0914 09:03:32.379617 140507706811584 submission.py:307] 66500) loss = 2.570, grad_norm = 0.500 +I0914 09:08:47.002616 140485041542912 logging_writer.py:48] [67000] global_step=67000, grad_norm=0.5, loss=4.07414 +I0914 09:08:47.006540 140507706811584 submission.py:307] 67000) loss = 4.074, grad_norm = 0.500 +I0914 09:17:37.231642 140484828653312 logging_writer.py:48] [67500] global_step=67500, grad_norm=0.5, loss=1.84739 +I0914 09:17:37.236508 140507706811584 submission.py:307] 67500) loss = 1.847, grad_norm = 0.500 +I0914 09:22:14.478449 140485041542912 logging_writer.py:48] [68000] global_step=68000, grad_norm=0.5, loss=1.80384 +I0914 09:22:14.482540 140507706811584 submission.py:307] 68000) loss = 1.804, grad_norm = 0.500 +I0914 09:29:09.392706 140484828653312 logging_writer.py:48] [68500] global_step=68500, grad_norm=0.5, loss=3.74476 +I0914 09:29:09.398597 140507706811584 submission.py:307] 68500) loss = 3.745, grad_norm = 0.500 +I0914 09:34:03.900097 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 09:34:55.242740 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 09:36:00.123647 140507706811584 spec.py:363] Evaluating on the test split. +I0914 09:36:01.072996 140507706811584 submission_runner.py:516] Time since start: 57763.05s, Step: 68773, {'train/accuracy': 0.6909765625, 'train/loss': 2.595083923339844, 'validation/accuracy': 0.63086, 'validation/loss': 2.31974203125, 'validation/num_examples': 50000, 'test/accuracy': 0.6061, 'test/loss': 1.7303037109375, 'test/num_examples': 10000, 'score': 54015.92000699043, 'total_duration': 57763.051216840744, 'accumulated_submission_time': 54015.92000699043, 'accumulated_eval_time': 3544.751893043518, 'accumulated_logging_time': 7.557418584823608} +I0914 09:36:01.388132 140485016364800 logging_writer.py:48] [68773] accumulated_eval_time=3544.75, accumulated_logging_time=7.55742, accumulated_submission_time=54015.9, global_step=68773, preemption_count=0, score=54015.9, test/accuracy=0.6061, test/loss=1.7303, test/num_examples=10000, total_duration=57763.1, train/accuracy=0.690977, train/loss=2.59508, validation/accuracy=0.63086, validation/loss=2.31974, validation/num_examples=50000 +I0914 09:39:00.606691 140485033150208 logging_writer.py:48] [69000] global_step=69000, grad_norm=0.5, loss=1.55964 +I0914 09:39:00.612642 140507706811584 submission.py:307] 69000) loss = 1.560, grad_norm = 0.500 +I0914 09:45:58.553005 140485016364800 logging_writer.py:48] [69500] global_step=69500, grad_norm=0.5, loss=3.9182 +I0914 09:45:58.558112 140507706811584 submission.py:307] 69500) loss = 3.918, grad_norm = 0.500 +I0914 09:55:02.126298 140485033150208 logging_writer.py:48] [70000] global_step=70000, grad_norm=0.5, loss=2.36073 +I0914 09:55:02.130255 140507706811584 submission.py:307] 70000) loss = 2.361, grad_norm = 0.500 +I0914 09:59:31.014065 140485016364800 logging_writer.py:48] [70500] global_step=70500, grad_norm=0.5, loss=2.59856 +I0914 09:59:31.019096 140507706811584 submission.py:307] 70500) loss = 2.599, grad_norm = 0.500 +I0914 10:06:54.576431 140485033150208 logging_writer.py:48] [71000] global_step=71000, grad_norm=0.5, loss=1.9473 +I0914 10:06:54.580720 140507706811584 submission.py:307] 71000) loss = 1.947, grad_norm = 0.500 +I0914 10:14:05.540501 140485016364800 logging_writer.py:48] [71500] global_step=71500, grad_norm=0.5, loss=1.50889 +I0914 10:14:05.547118 140507706811584 submission.py:307] 71500) loss = 1.509, grad_norm = 0.500 +I0914 10:18:57.269018 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 10:20:14.490741 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 10:21:19.918390 140507706811584 spec.py:363] Evaluating on the test split. +I0914 10:21:20.885500 140507706811584 submission_runner.py:516] Time since start: 60482.86s, Step: 71975, {'train/accuracy': 0.6887890625, 'train/loss': 2.768858642578125, 'validation/accuracy': 0.63156, 'validation/loss': 2.42385984375, 'validation/num_examples': 50000, 'test/accuracy': 0.6106, 'test/loss': 1.7244625, 'test/num_examples': 10000, 'score': 56582.710934877396, 'total_duration': 60482.86300563812, 'accumulated_submission_time': 56582.710934877396, 'accumulated_eval_time': 3688.368086576462, 'accumulated_logging_time': 7.884807109832764} +I0914 10:21:21.173105 140485091899136 logging_writer.py:48] [71975] accumulated_eval_time=3688.37, accumulated_logging_time=7.88481, accumulated_submission_time=56582.7, global_step=71975, preemption_count=0, score=56582.7, test/accuracy=0.6106, test/loss=1.72446, test/num_examples=10000, total_duration=60482.9, train/accuracy=0.688789, train/loss=2.76886, validation/accuracy=0.63156, validation/loss=2.42386, validation/num_examples=50000 +I0914 10:21:28.543692 140485007972096 logging_writer.py:48] [72000] global_step=72000, grad_norm=0.5, loss=2.12379 +I0914 10:21:28.547159 140507706811584 submission.py:307] 72000) loss = 2.124, grad_norm = 0.500 +I0914 10:29:48.708868 140485091899136 logging_writer.py:48] [72500] global_step=72500, grad_norm=0.5, loss=1.69216 +I0914 10:29:48.712683 140507706811584 submission.py:307] 72500) loss = 1.692, grad_norm = 0.500 +I0914 10:34:51.642090 140485007972096 logging_writer.py:48] [73000] global_step=73000, grad_norm=0.5, loss=3.41101 +I0914 10:34:51.646455 140507706811584 submission.py:307] 73000) loss = 3.411, grad_norm = 0.500 +I0914 10:42:05.829747 140485091899136 logging_writer.py:48] [73500] global_step=73500, grad_norm=0.5, loss=1.71274 +I0914 10:42:05.835457 140507706811584 submission.py:307] 73500) loss = 1.713, grad_norm = 0.500 +I0914 10:48:56.589232 140485007972096 logging_writer.py:48] [74000] global_step=74000, grad_norm=0.5, loss=2.06937 +I0914 10:48:56.603627 140507706811584 submission.py:307] 74000) loss = 2.069, grad_norm = 0.500 +I0914 10:54:04.485645 140485091899136 logging_writer.py:48] [74500] global_step=74500, grad_norm=0.5, loss=2.26177 +I0914 10:54:04.489653 140507706811584 submission.py:307] 74500) loss = 2.262, grad_norm = 0.500 +I0914 11:02:41.076665 140485007972096 logging_writer.py:48] [75000] global_step=75000, grad_norm=0.5, loss=2.49524 +I0914 11:02:41.082876 140507706811584 submission.py:307] 75000) loss = 2.495, grad_norm = 0.500 +I0914 11:04:16.917864 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 11:05:31.214393 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 11:06:36.624001 140507706811584 spec.py:363] Evaluating on the test split. +I0914 11:06:37.528402 140507706811584 submission_runner.py:516] Time since start: 63199.51s, Step: 75149, {'train/accuracy': 0.73109375, 'train/loss': 2.12315185546875, 'validation/accuracy': 0.61068, 'validation/loss': 2.7683290625, 'validation/num_examples': 50000, 'test/accuracy': 0.6132, 'test/loss': 1.722033203125, 'test/num_examples': 10000, 'score': 59149.80621433258, 'total_duration': 63199.50657391548, 'accumulated_submission_time': 59149.80621433258, 'accumulated_eval_time': 3828.978727579117, 'accumulated_logging_time': 8.191806316375732} +I0914 11:06:37.875280 140485117077248 logging_writer.py:48] [75149] accumulated_eval_time=3828.98, accumulated_logging_time=8.19181, accumulated_submission_time=59149.8, global_step=75149, preemption_count=0, score=59149.8, test/accuracy=0.6132, test/loss=1.72203, test/num_examples=10000, total_duration=63199.5, train/accuracy=0.731094, train/loss=2.12315, validation/accuracy=0.61068, validation/loss=2.76833, validation/num_examples=50000 +I0914 11:11:05.711035 140484999579392 logging_writer.py:48] [75500] global_step=75500, grad_norm=0.5, loss=1.44321 +I0914 11:11:05.716107 140507706811584 submission.py:307] 75500) loss = 1.443, grad_norm = 0.500 +I0914 11:19:22.184468 140485117077248 logging_writer.py:48] [76000] global_step=76000, grad_norm=0.5, loss=1.27353 +I0914 11:19:22.188590 140507706811584 submission.py:307] 76000) loss = 1.274, grad_norm = 0.500 +I0914 11:26:39.018251 140484999579392 logging_writer.py:48] [76500] global_step=76500, grad_norm=0.5, loss=1.47475 +I0914 11:26:39.029775 140507706811584 submission.py:307] 76500) loss = 1.475, grad_norm = 0.500 +I0914 11:31:45.079613 140485117077248 logging_writer.py:48] [77000] global_step=77000, grad_norm=0.5, loss=1.56408 +I0914 11:31:45.085461 140507706811584 submission.py:307] 77000) loss = 1.564, grad_norm = 0.500 +I0914 11:40:50.829887 140484999579392 logging_writer.py:48] [77500] global_step=77500, grad_norm=0.5, loss=1.63346 +I0914 11:40:50.834627 140507706811584 submission.py:307] 77500) loss = 1.633, grad_norm = 0.500 +I0914 11:45:20.949659 140485117077248 logging_writer.py:48] [78000] global_step=78000, grad_norm=0.5, loss=1.43918 +I0914 11:45:20.954147 140507706811584 submission.py:307] 78000) loss = 1.439, grad_norm = 0.500 +I0914 11:49:35.283980 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 11:50:59.421541 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 11:52:04.299040 140507706811584 spec.py:363] Evaluating on the test split. +I0914 11:52:05.277798 140507706811584 submission_runner.py:516] Time since start: 65927.25s, Step: 78343, {'train/accuracy': 0.686171875, 'train/loss': 2.8263055419921876, 'validation/accuracy': 0.62296, 'validation/loss': 2.597374375, 'validation/num_examples': 50000, 'test/accuracy': 0.6155, 'test/loss': 1.707414453125, 'test/num_examples': 10000, 'score': 61718.4423429966, 'total_duration': 65927.25494980812, 'accumulated_submission_time': 61718.4423429966, 'accumulated_eval_time': 3978.9714126586914, 'accumulated_logging_time': 8.550678730010986} +I0914 11:52:05.551152 140485007972096 logging_writer.py:48] [78343] accumulated_eval_time=3978.97, accumulated_logging_time=8.55068, accumulated_submission_time=61718.4, global_step=78343, preemption_count=0, score=61718.4, test/accuracy=0.6155, test/loss=1.70741, test/num_examples=10000, total_duration=65927.3, train/accuracy=0.686172, train/loss=2.82631, validation/accuracy=0.62296, validation/loss=2.59737, validation/num_examples=50000 +I0914 11:53:42.663558 140488370267904 logging_writer.py:48] [78500] global_step=78500, grad_norm=0.5, loss=3.63403 +I0914 11:53:42.667221 140507706811584 submission.py:307] 78500) loss = 3.634, grad_norm = 0.500 +I0914 12:01:10.499933 140485007972096 logging_writer.py:48] [79000] global_step=79000, grad_norm=0.5, loss=1.53774 +I0914 12:01:10.505761 140507706811584 submission.py:307] 79000) loss = 1.538, grad_norm = 0.500 +I0914 12:06:35.636253 140488370267904 logging_writer.py:48] [79500] global_step=79500, grad_norm=0.5, loss=1.38847 +I0914 12:06:35.640166 140507706811584 submission.py:307] 79500) loss = 1.388, grad_norm = 0.500 +I0914 12:15:14.641458 140485007972096 logging_writer.py:48] [80000] global_step=80000, grad_norm=0.5, loss=1.43119 +I0914 12:15:14.645396 140507706811584 submission.py:307] 80000) loss = 1.431, grad_norm = 0.500 +I0914 12:19:44.568130 140488370267904 logging_writer.py:48] [80500] global_step=80500, grad_norm=0.5, loss=3.8279 +I0914 12:19:44.572558 140507706811584 submission.py:307] 80500) loss = 3.828, grad_norm = 0.500 +I0914 12:26:39.132767 140485007972096 logging_writer.py:48] [81000] global_step=81000, grad_norm=0.5, loss=1.52316 +I0914 12:26:39.136672 140507706811584 submission.py:307] 81000) loss = 1.523, grad_norm = 0.500 +I0914 12:33:38.998826 140488370267904 logging_writer.py:48] [81500] global_step=81500, grad_norm=0.5, loss=1.26651 +I0914 12:33:39.029968 140507706811584 submission.py:307] 81500) loss = 1.267, grad_norm = 0.500 +I0914 12:35:01.941333 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 12:36:10.082504 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 12:37:14.994270 140507706811584 spec.py:363] Evaluating on the test split. +I0914 12:37:15.869337 140507706811584 submission_runner.py:516] Time since start: 68637.85s, Step: 81658, {'train/accuracy': 0.71037109375, 'train/loss': 2.5062692260742185, 'validation/accuracy': 0.64238, 'validation/loss': 2.26289203125, 'validation/num_examples': 50000, 'test/accuracy': 0.6195, 'test/loss': 1.6986482421875, 'test/num_examples': 10000, 'score': 64286.231412410736, 'total_duration': 68637.84749555588, 'accumulated_submission_time': 64286.231412410736, 'accumulated_eval_time': 4112.900275707245, 'accumulated_logging_time': 8.836618423461914} +I0914 12:37:16.161242 140485125469952 logging_writer.py:48] [81658] accumulated_eval_time=4112.9, accumulated_logging_time=8.83662, accumulated_submission_time=64286.2, global_step=81658, preemption_count=0, score=64286.2, test/accuracy=0.6195, test/loss=1.69865, test/num_examples=10000, total_duration=68637.8, train/accuracy=0.710371, train/loss=2.50627, validation/accuracy=0.64238, validation/loss=2.26289, validation/num_examples=50000 +I0914 12:41:57.584804 140485083506432 logging_writer.py:48] [82000] global_step=82000, grad_norm=0.5, loss=1.37238 +I0914 12:41:57.590211 140507706811584 submission.py:307] 82000) loss = 1.372, grad_norm = 0.500 +I0914 12:51:02.255522 140485125469952 logging_writer.py:48] [82500] global_step=82500, grad_norm=0.5, loss=1.41472 +I0914 12:51:02.260458 140507706811584 submission.py:307] 82500) loss = 1.415, grad_norm = 0.500 +I0914 12:55:45.043859 140485083506432 logging_writer.py:48] [83000] global_step=83000, grad_norm=0.5, loss=1.35322 +I0914 12:55:45.048648 140507706811584 submission.py:307] 83000) loss = 1.353, grad_norm = 0.500 +I0914 13:02:49.979186 140485125469952 logging_writer.py:48] [83500] global_step=83500, grad_norm=0.5, loss=1.33001 +I0914 13:02:49.986246 140507706811584 submission.py:307] 83500) loss = 1.330, grad_norm = 0.500 +I0914 13:09:58.072855 140485083506432 logging_writer.py:48] [84000] global_step=84000, grad_norm=0.5, loss=3.46984 +I0914 13:09:58.082865 140507706811584 submission.py:307] 84000) loss = 3.470, grad_norm = 0.500 +I0914 13:15:04.410156 140485125469952 logging_writer.py:48] [84500] global_step=84500, grad_norm=0.5, loss=1.38972 +I0914 13:15:04.414316 140507706811584 submission.py:307] 84500) loss = 1.390, grad_norm = 0.500 +I0914 13:20:19.622749 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 13:21:43.145435 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 13:22:48.405091 140507706811584 spec.py:363] Evaluating on the test split. +I0914 13:22:49.332628 140507706811584 submission_runner.py:516] Time since start: 71371.31s, Step: 84818, {'train/accuracy': 0.72197265625, 'train/loss': 2.4494094848632812, 'validation/accuracy': 0.61692, 'validation/loss': 2.79200625, 'validation/num_examples': 50000, 'test/accuracy': 0.6166, 'test/loss': 1.700314453125, 'test/num_examples': 10000, 'score': 66861.53665137291, 'total_duration': 71371.31085371971, 'accumulated_submission_time': 66861.53665137291, 'accumulated_eval_time': 4262.610267877579, 'accumulated_logging_time': 9.137850284576416} +I0914 13:22:49.597188 140485041542912 logging_writer.py:48] [84818] accumulated_eval_time=4262.61, accumulated_logging_time=9.13785, accumulated_submission_time=66861.5, global_step=84818, preemption_count=0, score=66861.5, test/accuracy=0.6166, test/loss=1.70031, test/num_examples=10000, total_duration=71371.3, train/accuracy=0.721973, train/loss=2.44941, validation/accuracy=0.61692, validation/loss=2.79201, validation/num_examples=50000 +I0914 13:25:11.469893 140485091899136 logging_writer.py:48] [85000] global_step=85000, grad_norm=0.5, loss=1.71786 +I0914 13:25:11.485686 140507706811584 submission.py:307] 85000) loss = 1.718, grad_norm = 0.500 +I0914 13:30:58.928760 140485041542912 logging_writer.py:48] [85500] global_step=85500, grad_norm=0.5, loss=2.54686 +I0914 13:30:58.933454 140507706811584 submission.py:307] 85500) loss = 2.547, grad_norm = 0.500 +I0914 13:38:27.179568 140485091899136 logging_writer.py:48] [86000] global_step=86000, grad_norm=0.5, loss=3.16338 +I0914 13:38:27.184804 140507706811584 submission.py:307] 86000) loss = 3.163, grad_norm = 0.500 +I0914 13:45:05.869021 140485041542912 logging_writer.py:48] [86500] global_step=86500, grad_norm=0.5, loss=1.86864 +I0914 13:45:05.873607 140507706811584 submission.py:307] 86500) loss = 1.869, grad_norm = 0.500 +I0914 13:50:10.162425 140485091899136 logging_writer.py:48] [87000] global_step=87000, grad_norm=0.5, loss=1.37481 +I0914 13:50:10.166693 140507706811584 submission.py:307] 87000) loss = 1.375, grad_norm = 0.500 +I0914 13:58:57.118870 140485041542912 logging_writer.py:48] [87500] global_step=87500, grad_norm=0.5, loss=1.44445 +I0914 13:58:57.124395 140507706811584 submission.py:307] 87500) loss = 1.444, grad_norm = 0.500 +I0914 14:03:26.384844 140485091899136 logging_writer.py:48] [88000] global_step=88000, grad_norm=0.5, loss=3.74655 +I0914 14:03:26.389654 140507706811584 submission.py:307] 88000) loss = 3.747, grad_norm = 0.500 +I0914 14:05:46.698072 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 14:07:04.177813 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 14:08:09.412411 140507706811584 spec.py:363] Evaluating on the test split. +I0914 14:08:10.325278 140507706811584 submission_runner.py:516] Time since start: 74092.30s, Step: 88203, {'train/accuracy': 0.74865234375, 'train/loss': 2.119185791015625, 'validation/accuracy': 0.61966, 'validation/loss': 2.73540625, 'validation/num_examples': 50000, 'test/accuracy': 0.6215, 'test/loss': 1.683543359375, 'test/num_examples': 10000, 'score': 69429.64796209335, 'total_duration': 74092.30352163315, 'accumulated_submission_time': 69429.64796209335, 'accumulated_eval_time': 4406.237560510635, 'accumulated_logging_time': 9.414802074432373} +I0914 14:08:10.702883 140484853831424 logging_writer.py:48] [88203] accumulated_eval_time=4406.24, accumulated_logging_time=9.4148, accumulated_submission_time=69429.6, global_step=88203, preemption_count=0, score=69429.6, test/accuracy=0.6215, test/loss=1.68354, test/num_examples=10000, total_duration=74092.3, train/accuracy=0.748652, train/loss=2.11919, validation/accuracy=0.61966, validation/loss=2.73541, validation/num_examples=50000 +I0914 14:12:15.312587 140484999579392 logging_writer.py:48] [88500] global_step=88500, grad_norm=0.5, loss=3.55842 +I0914 14:12:15.317419 140507706811584 submission.py:307] 88500) loss = 3.558, grad_norm = 0.500 +I0914 14:19:44.762547 140484853831424 logging_writer.py:48] [89000] global_step=89000, grad_norm=0.5, loss=1.2544 +I0914 14:19:44.800870 140507706811584 submission.py:307] 89000) loss = 1.254, grad_norm = 0.500 +I0914 14:25:03.408800 140484999579392 logging_writer.py:48] [89500] global_step=89500, grad_norm=0.5, loss=1.41412 +I0914 14:25:03.415614 140507706811584 submission.py:307] 89500) loss = 1.414, grad_norm = 0.500 +I0914 14:33:53.783746 140484853831424 logging_writer.py:48] [90000] global_step=90000, grad_norm=0.5, loss=1.28966 +I0914 14:33:53.787723 140507706811584 submission.py:307] 90000) loss = 1.290, grad_norm = 0.500 +I0914 14:38:18.897946 140484999579392 logging_writer.py:48] [90500] global_step=90500, grad_norm=0.5, loss=1.70013 +I0914 14:38:18.903419 140507706811584 submission.py:307] 90500) loss = 1.700, grad_norm = 0.500 +I0914 14:45:11.037715 140484853831424 logging_writer.py:48] [91000] global_step=91000, grad_norm=0.5, loss=3.63936 +I0914 14:45:11.044745 140507706811584 submission.py:307] 91000) loss = 3.639, grad_norm = 0.500 +I0914 14:51:06.434349 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 14:52:17.920029 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 14:53:23.046366 140507706811584 spec.py:363] Evaluating on the test split. +I0914 14:53:23.970502 140507706811584 submission_runner.py:516] Time since start: 76805.95s, Step: 91363, {'train/accuracy': 0.71162109375, 'train/loss': 2.6787774658203123, 'validation/accuracy': 0.63354, 'validation/loss': 2.514245, 'validation/num_examples': 50000, 'test/accuracy': 0.6294, 'test/loss': 1.683933984375, 'test/num_examples': 10000, 'score': 71996.89573025703, 'total_duration': 76805.9487221241, 'accumulated_submission_time': 71996.89573025703, 'accumulated_eval_time': 4543.773989677429, 'accumulated_logging_time': 9.803289890289307} +I0914 14:53:24.280097 140484991186688 logging_writer.py:48] [91363] accumulated_eval_time=4543.77, accumulated_logging_time=9.80329, accumulated_submission_time=71996.9, global_step=91363, preemption_count=0, score=71996.9, test/accuracy=0.6294, test/loss=1.68393, test/num_examples=10000, total_duration=76805.9, train/accuracy=0.711621, train/loss=2.67878, validation/accuracy=0.63354, validation/loss=2.51424, validation/num_examples=50000 +I0914 14:54:26.773252 140485058328320 logging_writer.py:48] [91500] global_step=91500, grad_norm=0.5, loss=1.27041 +I0914 14:54:26.778402 140507706811584 submission.py:307] 91500) loss = 1.270, grad_norm = 0.500 +I0914 15:02:08.281780 140484991186688 logging_writer.py:48] [92000] global_step=92000, grad_norm=0.5, loss=2.44104 +I0914 15:02:08.288162 140507706811584 submission.py:307] 92000) loss = 2.441, grad_norm = 0.500 +I0914 15:11:21.171151 140485058328320 logging_writer.py:48] [92500] global_step=92500, grad_norm=0.5, loss=1.27221 +I0914 15:11:21.175150 140507706811584 submission.py:307] 92500) loss = 1.272, grad_norm = 0.500 +I0914 15:16:03.307185 140484991186688 logging_writer.py:48] [93000] global_step=93000, grad_norm=0.5, loss=1.3779 +I0914 15:16:03.312820 140507706811584 submission.py:307] 93000) loss = 1.378, grad_norm = 0.500 +I0914 15:23:15.317771 140485058328320 logging_writer.py:48] [93500] global_step=93500, grad_norm=0.5, loss=1.24184 +I0914 15:23:15.324241 140507706811584 submission.py:307] 93500) loss = 1.242, grad_norm = 0.500 +I0914 15:30:34.342267 140484991186688 logging_writer.py:48] [94000] global_step=94000, grad_norm=0.5, loss=1.20235 +I0914 15:30:34.372657 140507706811584 submission.py:307] 94000) loss = 1.202, grad_norm = 0.500 +I0914 15:35:25.647907 140485058328320 logging_writer.py:48] [94500] global_step=94500, grad_norm=0.5, loss=3.51255 +I0914 15:35:25.653179 140507706811584 submission.py:307] 94500) loss = 3.513, grad_norm = 0.500 +I0914 15:36:19.988276 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 15:37:46.294738 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 15:38:52.169045 140507706811584 spec.py:363] Evaluating on the test split. +I0914 15:38:53.210876 140507706811584 submission_runner.py:516] Time since start: 79535.19s, Step: 94562, {'train/accuracy': 0.75099609375, 'train/loss': 2.18218994140625, 'validation/accuracy': 0.63316, 'validation/loss': 2.51823375, 'validation/num_examples': 50000, 'test/accuracy': 0.6312, 'test/loss': 1.665711328125, 'test/num_examples': 10000, 'score': 74563.37644934654, 'total_duration': 79535.18826961517, 'accumulated_submission_time': 74563.37644934654, 'accumulated_eval_time': 4696.995683193207, 'accumulated_logging_time': 10.130064964294434} +I0914 15:38:53.544306 140485041542912 logging_writer.py:48] [94562] accumulated_eval_time=4697, accumulated_logging_time=10.1301, accumulated_submission_time=74563.4, global_step=94562, preemption_count=0, score=74563.4, test/accuracy=0.6312, test/loss=1.66571, test/num_examples=10000, total_duration=79535.2, train/accuracy=0.750996, train/loss=2.18219, validation/accuracy=0.63316, validation/loss=2.51823, validation/num_examples=50000 +I0914 15:45:43.294047 140488370267904 logging_writer.py:48] [95000] global_step=95000, grad_norm=0.5, loss=1.19887 +I0914 15:45:43.298103 140507706811584 submission.py:307] 95000) loss = 1.199, grad_norm = 0.500 +I0914 15:51:04.248950 140485041542912 logging_writer.py:48] [95500] global_step=95500, grad_norm=0.5, loss=1.3123 +I0914 15:51:04.253101 140507706811584 submission.py:307] 95500) loss = 1.312, grad_norm = 0.500 +I0914 15:58:09.693961 140488370267904 logging_writer.py:48] [96000] global_step=96000, grad_norm=0.5, loss=1.16824 +I0914 15:58:09.699718 140507706811584 submission.py:307] 96000) loss = 1.168, grad_norm = 0.500 +I0914 16:05:02.458126 140485041542912 logging_writer.py:48] [96500] global_step=96500, grad_norm=0.5, loss=1.24271 +I0914 16:05:02.464063 140507706811584 submission.py:307] 96500) loss = 1.243, grad_norm = 0.500 +I0914 16:10:05.472149 140488370267904 logging_writer.py:48] [97000] global_step=97000, grad_norm=0.5, loss=1.82541 +I0914 16:10:05.480813 140507706811584 submission.py:307] 97000) loss = 1.825, grad_norm = 0.500 +I0914 16:18:36.637383 140485041542912 logging_writer.py:48] [97500] global_step=97500, grad_norm=0.5, loss=1.49459 +I0914 16:18:36.652518 140507706811584 submission.py:307] 97500) loss = 1.495, grad_norm = 0.500 +I0914 16:21:48.728986 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 16:23:04.155695 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 16:24:09.161276 140507706811584 spec.py:363] Evaluating on the test split. +I0914 16:24:10.079079 140507706811584 submission_runner.py:516] Time since start: 82252.06s, Step: 97834, {'train/accuracy': 0.74978515625, 'train/loss': 2.2532208251953123, 'validation/accuracy': 0.64504, 'validation/loss': 2.42088640625, 'validation/num_examples': 50000, 'test/accuracy': 0.6259, 'test/loss': 1.68273125, 'test/num_examples': 10000, 'score': 77130.09559607506, 'total_duration': 82252.05650377274, 'accumulated_submission_time': 77130.09559607506, 'accumulated_eval_time': 4838.347635269165, 'accumulated_logging_time': 10.473679780960083} +I0914 16:24:10.433984 140485007972096 logging_writer.py:48] [97834] accumulated_eval_time=4838.35, accumulated_logging_time=10.4737, accumulated_submission_time=77130.1, global_step=97834, preemption_count=0, score=77130.1, test/accuracy=0.6259, test/loss=1.68273, test/num_examples=10000, total_duration=82252.1, train/accuracy=0.749785, train/loss=2.25322, validation/accuracy=0.64504, validation/loss=2.42089, validation/num_examples=50000 +I0914 16:25:52.226353 140485024757504 logging_writer.py:48] [98000] global_step=98000, grad_norm=0.5, loss=1.36966 +I0914 16:25:52.233252 140507706811584 submission.py:307] 98000) loss = 1.370, grad_norm = 0.500 +I0914 16:34:23.703118 140485007972096 logging_writer.py:48] [98500] global_step=98500, grad_norm=0.5, loss=1.14689 +I0914 16:34:23.709549 140507706811584 submission.py:307] 98500) loss = 1.147, grad_norm = 0.500 +I0914 16:41:54.575507 140485024757504 logging_writer.py:48] [99000] global_step=99000, grad_norm=0.5, loss=1.20528 +I0914 16:41:54.584301 140507706811584 submission.py:307] 99000) loss = 1.205, grad_norm = 0.500 +I0914 16:47:01.734953 140485007972096 logging_writer.py:48] [99500] global_step=99500, grad_norm=0.5, loss=2.09376 +I0914 16:47:01.739069 140507706811584 submission.py:307] 99500) loss = 2.094, grad_norm = 0.500 +I0914 16:55:54.309578 140485024757504 logging_writer.py:48] [100000] global_step=100000, grad_norm=0.5, loss=1.41589 +I0914 16:55:54.314973 140507706811584 submission.py:307] 100000) loss = 1.416, grad_norm = 0.500 +I0914 17:00:34.595384 140485007972096 logging_writer.py:48] [100500] global_step=100500, grad_norm=0.5, loss=1.38283 +I0914 17:00:34.599744 140507706811584 submission.py:307] 100500) loss = 1.383, grad_norm = 0.500 +I0914 17:07:07.375317 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 17:08:30.082203 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 17:09:35.541888 140507706811584 spec.py:363] Evaluating on the test split. +I0914 17:09:36.416068 140507706811584 submission_runner.py:516] Time since start: 84978.39s, Step: 100988, {'train/accuracy': 0.769765625, 'train/loss': 2.051152801513672, 'validation/accuracy': 0.64088, 'validation/loss': 2.5134090625, 'validation/num_examples': 50000, 'test/accuracy': 0.6316, 'test/loss': 1.67145859375, 'test/num_examples': 10000, 'score': 79698.34933590889, 'total_duration': 84978.39427757263, 'accumulated_submission_time': 79698.34933590889, 'accumulated_eval_time': 4987.388442993164, 'accumulated_logging_time': 10.837621450424194} +I0914 17:09:36.727324 140484999579392 logging_writer.py:48] [100988] accumulated_eval_time=4987.39, accumulated_logging_time=10.8376, accumulated_submission_time=79698.3, global_step=100988, preemption_count=0, score=79698.3, test/accuracy=0.6316, test/loss=1.67146, test/num_examples=10000, total_duration=84978.4, train/accuracy=0.769766, train/loss=2.05115, validation/accuracy=0.64088, validation/loss=2.51341, validation/num_examples=50000 +I0914 17:09:40.665591 140485024757504 logging_writer.py:48] [101000] global_step=101000, grad_norm=0.5, loss=1.38035 +I0914 17:09:40.668834 140507706811584 submission.py:307] 101000) loss = 1.380, grad_norm = 0.500 +I0914 17:16:22.962415 140484999579392 logging_writer.py:48] [101500] global_step=101500, grad_norm=0.5, loss=3.29463 +I0914 17:16:22.976444 140507706811584 submission.py:307] 101500) loss = 3.295, grad_norm = 0.500 +I0914 17:21:58.036333 140485024757504 logging_writer.py:48] [102000] global_step=102000, grad_norm=0.5, loss=3.06899 +I0914 17:21:58.040966 140507706811584 submission.py:307] 102000) loss = 3.069, grad_norm = 0.500 +I0914 17:30:17.468979 140484999579392 logging_writer.py:48] [102500] global_step=102500, grad_norm=0.5, loss=2.66818 +I0914 17:30:17.472806 140507706811584 submission.py:307] 102500) loss = 2.668, grad_norm = 0.500 +I0914 17:34:51.603332 140485024757504 logging_writer.py:48] [103000] global_step=103000, grad_norm=0.5, loss=1.37057 +I0914 17:34:51.612731 140507706811584 submission.py:307] 103000) loss = 1.371, grad_norm = 0.500 +I0914 17:41:48.970698 140484999579392 logging_writer.py:48] [103500] global_step=103500, grad_norm=0.5, loss=1.24068 +I0914 17:41:48.976109 140507706811584 submission.py:307] 103500) loss = 1.241, grad_norm = 0.500 +I0914 17:49:06.226054 140485024757504 logging_writer.py:48] [104000] global_step=104000, grad_norm=0.5, loss=1.84446 +I0914 17:49:06.240424 140507706811584 submission.py:307] 104000) loss = 1.844, grad_norm = 0.500 +I0914 17:52:31.995656 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 17:53:54.388982 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 17:54:59.817166 140507706811584 spec.py:363] Evaluating on the test split. +I0914 17:55:00.784630 140507706811584 submission_runner.py:516] Time since start: 87702.76s, Step: 104335, {'train/accuracy': 0.73486328125, 'train/loss': 2.56908935546875, 'validation/accuracy': 0.63392, 'validation/loss': 2.6882578125, 'validation/num_examples': 50000, 'test/accuracy': 0.6326, 'test/loss': 1.6640064453125, 'test/num_examples': 10000, 'score': 82264.89665937424, 'total_duration': 87702.76287913322, 'accumulated_submission_time': 82264.89665937424, 'accumulated_eval_time': 5136.17781662941, 'accumulated_logging_time': 11.158146619796753} +I0914 17:55:01.190526 140485075113728 logging_writer.py:48] [104335] accumulated_eval_time=5136.18, accumulated_logging_time=11.1581, accumulated_submission_time=82264.9, global_step=104335, preemption_count=0, score=82264.9, test/accuracy=0.6326, test/loss=1.66401, test/num_examples=10000, total_duration=87702.8, train/accuracy=0.734863, train/loss=2.56909, validation/accuracy=0.63392, validation/loss=2.68826, validation/num_examples=50000 +I0914 17:56:56.498791 140484982793984 logging_writer.py:48] [104500] global_step=104500, grad_norm=0.5, loss=1.21133 +I0914 17:56:56.503313 140507706811584 submission.py:307] 104500) loss = 1.211, grad_norm = 0.500 +I0914 18:07:01.756009 140485075113728 logging_writer.py:48] [105000] global_step=105000, grad_norm=0.5, loss=1.791 +I0914 18:07:01.759825 140507706811584 submission.py:307] 105000) loss = 1.791, grad_norm = 0.500 +I0914 18:12:12.314617 140484982793984 logging_writer.py:48] [105500] global_step=105500, grad_norm=0.5, loss=2.56654 +I0914 18:12:12.318906 140507706811584 submission.py:307] 105500) loss = 2.567, grad_norm = 0.500 +I0914 18:19:16.240788 140485075113728 logging_writer.py:48] [106000] global_step=106000, grad_norm=0.5, loss=1.12769 +I0914 18:19:16.246190 140507706811584 submission.py:307] 106000) loss = 1.128, grad_norm = 0.500 +I0914 18:26:39.342822 140484982793984 logging_writer.py:48] [106500] global_step=106500, grad_norm=0.5, loss=1.18743 +I0914 18:26:39.348372 140507706811584 submission.py:307] 106500) loss = 1.187, grad_norm = 0.500 +I0914 18:31:40.630032 140485075113728 logging_writer.py:48] [107000] global_step=107000, grad_norm=0.5, loss=1.1014 +I0914 18:31:40.636525 140507706811584 submission.py:307] 107000) loss = 1.101, grad_norm = 0.500 +I0914 18:38:00.043737 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 18:39:19.618727 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 18:40:25.218562 140507706811584 spec.py:363] Evaluating on the test split. +I0914 18:40:26.065144 140507706811584 submission_runner.py:516] Time since start: 90428.04s, Step: 107376, {'train/accuracy': 0.7356640625, 'train/loss': 2.609210205078125, 'validation/accuracy': 0.65224, 'validation/loss': 2.36879171875, 'validation/num_examples': 50000, 'test/accuracy': 0.6349, 'test/loss': 1.67556484375, 'test/num_examples': 10000, 'score': 84834.94648170471, 'total_duration': 90428.04349565506, 'accumulated_submission_time': 84834.94648170471, 'accumulated_eval_time': 5282.199355840683, 'accumulated_logging_time': 11.576345443725586} +I0914 18:40:26.437328 140485091899136 logging_writer.py:48] [107376] accumulated_eval_time=5282.2, accumulated_logging_time=11.5763, accumulated_submission_time=84834.9, global_step=107376, preemption_count=0, score=84834.9, test/accuracy=0.6349, test/loss=1.67556, test/num_examples=10000, total_duration=90428, train/accuracy=0.735664, train/loss=2.60921, validation/accuracy=0.65224, validation/loss=2.36879, validation/num_examples=50000 +I0914 18:41:39.112017 140485117077248 logging_writer.py:48] [107500] global_step=107500, grad_norm=0.5, loss=1.24864 +I0914 18:41:39.115706 140507706811584 submission.py:307] 107500) loss = 1.249, grad_norm = 0.500 +I0914 18:47:42.033792 140485091899136 logging_writer.py:48] [108000] global_step=108000, grad_norm=0.5, loss=2.28607 +I0914 18:47:42.038987 140507706811584 submission.py:307] 108000) loss = 2.286, grad_norm = 0.500 +I0914 18:55:08.256778 140485117077248 logging_writer.py:48] [108500] global_step=108500, grad_norm=0.5, loss=1.17154 +I0914 18:55:08.266123 140507706811584 submission.py:307] 108500) loss = 1.172, grad_norm = 0.500 +I0914 19:02:10.003679 140485091899136 logging_writer.py:48] [109000] global_step=109000, grad_norm=0.5, loss=1.16341 +I0914 19:02:10.007842 140507706811584 submission.py:307] 109000) loss = 1.163, grad_norm = 0.500 +I0914 19:07:07.029821 140485117077248 logging_writer.py:48] [109500] global_step=109500, grad_norm=0.5, loss=1.17068 +I0914 19:07:07.034625 140507706811584 submission.py:307] 109500) loss = 1.171, grad_norm = 0.500 +I0914 19:15:54.088518 140485091899136 logging_writer.py:48] [110000] global_step=110000, grad_norm=0.5, loss=1.8875 +I0914 19:15:54.094202 140507706811584 submission.py:307] 110000) loss = 1.887, grad_norm = 0.500 +I0914 19:20:39.990430 140485117077248 logging_writer.py:48] [110500] global_step=110500, grad_norm=0.5, loss=1.07768 +I0914 19:20:39.994623 140507706811584 submission.py:307] 110500) loss = 1.078, grad_norm = 0.500 +I0914 19:23:22.598462 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 19:24:38.991286 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 19:25:44.545756 140507706811584 spec.py:363] Evaluating on the test split. +I0914 19:25:45.436861 140507706811584 submission_runner.py:516] Time since start: 93147.41s, Step: 110729, {'train/accuracy': 0.73681640625, 'train/loss': 2.6602178955078126, 'validation/accuracy': 0.64202, 'validation/loss': 2.557555, 'validation/num_examples': 50000, 'test/accuracy': 0.6334, 'test/loss': 1.69276484375, 'test/num_examples': 10000, 'score': 87402.47606682777, 'total_duration': 93147.41487288475, 'accumulated_submission_time': 87402.47606682777, 'accumulated_eval_time': 5425.037556171417, 'accumulated_logging_time': 11.958082914352417} +I0914 19:25:45.853886 140485125469952 logging_writer.py:48] [110729] accumulated_eval_time=5425.04, accumulated_logging_time=11.9581, accumulated_submission_time=87402.5, global_step=110729, preemption_count=0, score=87402.5, test/accuracy=0.6334, test/loss=1.69276, test/num_examples=10000, total_duration=93147.4, train/accuracy=0.736816, train/loss=2.66022, validation/accuracy=0.64202, validation/loss=2.55755, validation/num_examples=50000 +I0914 19:29:28.064370 140485108684544 logging_writer.py:48] [111000] global_step=111000, grad_norm=0.5, loss=1.1404 +I0914 19:29:28.068273 140507706811584 submission.py:307] 111000) loss = 1.140, grad_norm = 0.500 +I0914 19:36:59.734897 140485125469952 logging_writer.py:48] [111500] global_step=111500, grad_norm=0.5, loss=1.27967 +I0914 19:36:59.740497 140507706811584 submission.py:307] 111500) loss = 1.280, grad_norm = 0.500 +I0914 19:42:13.925096 140485108684544 logging_writer.py:48] [112000] global_step=112000, grad_norm=0.5, loss=1.40026 +I0914 19:42:13.929051 140507706811584 submission.py:307] 112000) loss = 1.400, grad_norm = 0.500 +I0914 19:50:48.520788 140485125469952 logging_writer.py:48] [112500] global_step=112500, grad_norm=0.5, loss=1.16155 +I0914 19:50:48.528859 140507706811584 submission.py:307] 112500) loss = 1.162, grad_norm = 0.500 +I0914 19:55:42.838520 140485108684544 logging_writer.py:48] [113000] global_step=113000, grad_norm=0.5, loss=1.16817 +I0914 19:55:42.843236 140507706811584 submission.py:307] 113000) loss = 1.168, grad_norm = 0.500 +I0914 20:02:23.362958 140485125469952 logging_writer.py:48] [113500] global_step=113500, grad_norm=0.5, loss=2.72364 +I0914 20:02:23.366846 140507706811584 submission.py:307] 113500) loss = 2.724, grad_norm = 0.500 +I0914 20:08:43.097831 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 20:09:52.624787 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 20:10:57.392909 140507706811584 spec.py:363] Evaluating on the test split. +I0914 20:10:58.367114 140507706811584 submission_runner.py:516] Time since start: 95860.35s, Step: 113881, {'train/accuracy': 0.78509765625, 'train/loss': 1.9999606323242187, 'validation/accuracy': 0.6453, 'validation/loss': 2.511811875, 'validation/num_examples': 50000, 'test/accuracy': 0.6341, 'test/loss': 1.6815181640625, 'test/num_examples': 10000, 'score': 89971.41382741928, 'total_duration': 95860.34528636932, 'accumulated_submission_time': 89971.41382741928, 'accumulated_eval_time': 5560.306791305542, 'accumulated_logging_time': 12.384655714035034} +I0914 20:10:58.683879 140485083506432 logging_writer.py:48] [113881] accumulated_eval_time=5560.31, accumulated_logging_time=12.3847, accumulated_submission_time=89971.4, global_step=113881, preemption_count=0, score=89971.4, test/accuracy=0.6341, test/loss=1.68152, test/num_examples=10000, total_duration=95860.3, train/accuracy=0.785098, train/loss=1.99996, validation/accuracy=0.6453, validation/loss=2.51181, validation/num_examples=50000 +I0914 20:11:41.362562 140484999579392 logging_writer.py:48] [114000] global_step=114000, grad_norm=0.5, loss=1.18916 +I0914 20:11:41.366272 140507706811584 submission.py:307] 114000) loss = 1.189, grad_norm = 0.500 +I0914 20:19:13.340281 140485083506432 logging_writer.py:48] [114500] global_step=114500, grad_norm=0.5, loss=1.33549 +I0914 20:19:13.344320 140507706811584 submission.py:307] 114500) loss = 1.335, grad_norm = 0.500 +I0914 20:28:19.916662 140484999579392 logging_writer.py:48] [115000] global_step=115000, grad_norm=0.5, loss=3.36803 +I0914 20:28:19.920593 140507706811584 submission.py:307] 115000) loss = 3.368, grad_norm = 0.500 +I0914 20:33:19.826413 140485083506432 logging_writer.py:48] [115500] global_step=115500, grad_norm=0.5, loss=2.04045 +I0914 20:33:19.833363 140507706811584 submission.py:307] 115500) loss = 2.040, grad_norm = 0.500 +I0914 20:40:21.477324 140484999579392 logging_writer.py:48] [116000] global_step=116000, grad_norm=0.5, loss=1.02247 +I0914 20:40:21.483355 140507706811584 submission.py:307] 116000) loss = 1.022, grad_norm = 0.500 +I0914 20:47:46.726125 140485083506432 logging_writer.py:48] [116500] global_step=116500, grad_norm=0.5, loss=3.16979 +I0914 20:47:46.732438 140507706811584 submission.py:307] 116500) loss = 3.170, grad_norm = 0.500 +I0914 20:52:37.735978 140484999579392 logging_writer.py:48] [117000] global_step=117000, grad_norm=0.5, loss=2.13738 +I0914 20:52:37.739855 140507706811584 submission.py:307] 117000) loss = 2.137, grad_norm = 0.500 +I0914 20:53:55.435363 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 20:55:17.574085 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 20:56:23.266561 140507706811584 spec.py:363] Evaluating on the test split. +I0914 20:56:24.283072 140507706811584 submission_runner.py:516] Time since start: 98586.26s, Step: 117085, {'train/accuracy': 0.78712890625, 'train/loss': 2.0392381286621095, 'validation/accuracy': 0.66552, 'validation/loss': 2.1756734375, 'validation/num_examples': 50000, 'test/accuracy': 0.6389, 'test/loss': 1.67883984375, 'test/num_examples': 10000, 'score': 92539.17739653587, 'total_duration': 98586.26121616364, 'accumulated_submission_time': 92539.17739653587, 'accumulated_eval_time': 5709.154621601105, 'accumulated_logging_time': 12.710777521133423} +I0914 20:56:24.673391 140485007972096 logging_writer.py:48] [117085] accumulated_eval_time=5709.15, accumulated_logging_time=12.7108, accumulated_submission_time=92539.2, global_step=117085, preemption_count=0, score=92539.2, test/accuracy=0.6389, test/loss=1.67884, test/num_examples=10000, total_duration=98586.3, train/accuracy=0.787129, train/loss=2.03924, validation/accuracy=0.66552, validation/loss=2.17567, validation/num_examples=50000 +I0914 21:02:51.015796 140485066721024 logging_writer.py:48] [117500] global_step=117500, grad_norm=0.5, loss=2.14263 +I0914 21:02:51.022092 140507706811584 submission.py:307] 117500) loss = 2.143, grad_norm = 0.500 +I0914 21:08:13.620776 140485007972096 logging_writer.py:48] [118000] global_step=118000, grad_norm=0.5, loss=3.23044 +I0914 21:08:13.627437 140507706811584 submission.py:307] 118000) loss = 3.230, grad_norm = 0.500 +I0914 21:15:00.310102 140485066721024 logging_writer.py:48] [118500] global_step=118500, grad_norm=0.5, loss=3.23172 +I0914 21:15:00.314054 140507706811584 submission.py:307] 118500) loss = 3.232, grad_norm = 0.500 +I0914 21:22:08.883921 140485007972096 logging_writer.py:48] [119000] global_step=119000, grad_norm=0.5, loss=1.03288 +I0914 21:22:08.919276 140507706811584 submission.py:307] 119000) loss = 1.033, grad_norm = 0.500 +I0914 21:27:01.206306 140485066721024 logging_writer.py:48] [119500] global_step=119500, grad_norm=0.5, loss=0.921238 +I0914 21:27:01.211759 140507706811584 submission.py:307] 119500) loss = 0.921, grad_norm = 0.500 +I0914 21:35:20.921284 140485007972096 logging_writer.py:48] [120000] global_step=120000, grad_norm=0.5, loss=1.38743 +I0914 21:35:20.925400 140507706811584 submission.py:307] 120000) loss = 1.387, grad_norm = 0.500 +I0914 21:39:22.105983 140507706811584 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 21:40:29.087309 140507706811584 spec.py:346] Evaluating on the validation split. +I0914 21:41:34.615737 140507706811584 spec.py:363] Evaluating on the test split. +I0914 21:41:35.568121 140507706811584 submission_runner.py:516] Time since start: 101297.55s, Step: 120398, {'train/accuracy': 0.78953125, 'train/loss': 2.175392761230469, 'validation/accuracy': 0.63462, 'validation/loss': 2.829705, 'validation/num_examples': 50000, 'test/accuracy': 0.6362, 'test/loss': 1.70942421875, 'test/num_examples': 10000, 'score': 95106.80141806602, 'total_duration': 101297.54602193832, 'accumulated_submission_time': 95106.80141806602, 'accumulated_eval_time': 5842.61648774147, 'accumulated_logging_time': 13.123896360397339} +I0914 21:41:35.981124 140484999579392 logging_writer.py:48] [120398] accumulated_eval_time=5842.62, accumulated_logging_time=13.1239, accumulated_submission_time=95106.8, global_step=120398, preemption_count=0, score=95106.8, test/accuracy=0.6362, test/loss=1.70942, test/num_examples=10000, total_duration=101298, train/accuracy=0.789531, train/loss=2.17539, validation/accuracy=0.63462, validation/loss=2.82971, validation/num_examples=50000 +I0914 21:42:15.409630 140485033150208 logging_writer.py:48] [120500] global_step=120500, grad_norm=0.5, loss=2.7533 +I0914 21:42:15.414495 140507706811584 submission.py:307] 120500) loss = 2.753, grad_norm = 0.500 +I0914 21:50:46.713740 140484999579392 logging_writer.py:48] [121000] global_step=121000, grad_norm=0.5, loss=0.933754 +I0914 21:50:46.718786 140507706811584 submission.py:307] 121000) loss = 0.934, grad_norm = 0.500 +I0914 21:58:32.403998 140485033150208 logging_writer.py:48] [121500] global_step=121500, grad_norm=0.5, loss=1.00378 +I0914 21:58:32.409058 140507706811584 submission.py:307] 121500) loss = 1.004, grad_norm = 0.500 +I0914 22:03:27.341537 140484999579392 logging_writer.py:48] [122000] global_step=122000, grad_norm=0.5, loss=0.979969 +I0914 22:03:27.345991 140507706811584 submission.py:307] 122000) loss = 0.980, grad_norm = 0.500 +I0914 22:12:08.783092 140485033150208 logging_writer.py:48] [122500] global_step=122500, grad_norm=0.5, loss=3.36847 +I0914 22:12:08.789137 140507706811584 submission.py:307] 122500) loss = 3.368, grad_norm = 0.500 +I0914 22:17:08.294881 140484999579392 logging_writer.py:48] [123000] global_step=123000, grad_norm=0.5, loss=3.11381 +I0914 22:17:08.299254 140507706811584 submission.py:307] 123000) loss = 3.114, grad_norm = 0.500 +I0914 22:23:35.975229 140485033150208 logging_writer.py:48] [123500] global_step=123500, grad_norm=0.5, loss=1.28115 +I0914 22:23:35.981756 140507706811584 submission.py:307] 123500) loss = 1.281, grad_norm = 0.500 +I0914 22:24:35.633069 140484999579392 logging_writer.py:48] [123557] global_step=123557, preemption_count=0, score=97681.8 +I0914 22:24:47.139685 140507706811584 submission_runner.py:857] Final imagenet_vit score: 97681.78158950806 diff --git a/logs/self_tuning/ademamix_golden/study_1/imagenet_vit_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_1/imagenet_vit_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..a45f65d8 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/imagenet_vit_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,39 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/accuracy,test/loss,test/num_examples,total_duration,train/accuracy,train/loss,validation/accuracy,validation/loss,validation/num_examples +548.3675837516785,0.0,95.80058550834656,1,0,95.80058550834656,0.0019,6.90775546875,10000,644.7618706226349,0.0023046875,6.90775634765625,0.00192,6.90775625,50000 +692.409998178482,0.0468320846557617,2665.01338338852,3623,0,2665.01338338852,0.0754,5.2725359375,10000,3373.422957658768,0.09013671875,5.483372802734375,0.08084,5.48525625,50000 +833.9168169498444,0.3999044895172119,5230.912830114365,7023,0,5230.912830114365,0.2128,4.00091328125,10000,6090.881366968155,0.25580078125,4.329656372070312,0.21272,4.58559375,50000 +970.6382474899292,0.7354223728179932,7797.297001361847,10375,0,7797.297001361847,0.313,3.3613,10000,8803.147234916687,0.37193359375,3.657047119140625,0.31606,4.042581875,50000 +1109.6197061538696,1.26928973197937,10368.7392745018,13533,0,10368.7392745018,0.3787,2.8829498046875,10000,11522.681096076964,0.451796875,3.357933349609375,0.38034,3.827965625,50000 +1254.5655632019043,1.678205966949463,12935.96284532547,16925,0,12935.96284532547,0.4293,2.6210931640625,10000,14245.187555074692,0.50466796875,3.1147216796875,0.4406,3.221945,50000 +1397.178419828415,2.48038101196289,15502.118244171144,20132,0,15502.118244171144,0.4636,2.4247896484375,10000,16962.870960712433,0.51876953125,3.346199645996094,0.48268,3.011945,50000 +1544.2523880004885,2.8307547569274902,18068.6707174778,23359,0,18068.6707174778,0.4881,2.2952814453125,10000,19685.892365455627,0.56783203125,2.9495895385742186,0.50794,3.08117125,50000 +1688.974037885666,3.12606143951416,20635.43046736717,26742,0,20635.43046736717,0.5129,2.1553640625,10000,22406.56116294861,0.60248046875,2.754895324707032,0.52264,3.144949375,50000 +1847.059274435044,3.481067657470703,23202.31759428978,29900,0,23202.31759428978,0.5292,2.0948826171875,10000,25140.786123991013,0.61646484375,2.706271057128906,0.53204,2.973585625,50000 +1992.120970964432,3.769123792648315,25768.509074926376,33252,0,25768.509074926376,0.543,2.029508984375,10000,27862.138606786728,0.6110546875,2.8116305541992186,0.55746,2.6731121875,50000 +2129.486294984817,4.115705490112305,28333.736892700195,36543,0,28333.736892700195,0.5498,1.9853494140625,10000,30575.47583174705,0.66943359375,2.203136749267578,0.5562,2.953885625,50000 +2283.681713104248,4.536134004592896,30901.9447760582,39689,0,30901.9447760582,0.5596,1.9409359375,10000,33307.08735227585,0.64908203125,2.658007507324218,0.5803,2.6447209375,50000 +2425.578112602234,4.859193801879883,33470.53742003441,43077,0,33470.53742003441,0.5687,1.913438671875,10000,36026.5645134449,0.65333984375,2.679707946777344,0.57702,2.819873125,50000 +2556.88010764122,5.248838663101196,36045.62726044655,46226,0,36045.62726044655,0.5766,1.855303515625,10000,38741.59304046631,0.66466796875,2.5644097900390626,0.58578,2.784920625,50000 +2704.4651052951813,5.631556510925293,38612.53212809563,49479,0,38612.53212809563,0.5787,1.857298828125,10000,41465.013504982,0.6749609375,2.5443516540527344,0.58796,2.8763503125,50000 +2843.9927847385406,5.925373315811157,41181.36528348923,52682,0,41181.36528348923,0.5899,1.833320703125,10000,44181.918159246445,0.67267578125,2.701242065429688,0.59154,2.840863125,50000 +2989.893040895462,6.267249822616577,43747.88288974762,55813,0,43747.88288974762,0.5934,1.80217890625,10000,46905.568497657776,0.6837109375,2.5283877563476564,0.61526,2.4480253125,50000 +3128.649794578552,6.634281158447266,46314.69552898407,59086,0,46314.69552898407,0.5995,1.7874669921875,10000,49620.04446363449,0.66517578125,2.8801678466796874,0.61194,2.4760378125,50000 +3285.323085308075,6.9809653759002686,48881.777411699295,62233,0,48881.777411699295,0.5972,1.782680859375,10000,52352.351593256,0.6954296875,2.445909576416016,0.6085,2.61531984375,50000 +3427.57888674736,7.273256301879883,51448.57416796684,65639,0,51448.57416796684,0.6034,1.7580052734375,10000,55070.69747138024,0.693125,2.52487060546875,0.61096,2.59159,50000 +3544.751893043518,7.557418584823608,54015.92000699043,68773,0,54015.92000699043,0.6061,1.7303037109375,10000,57763.05121684074,0.6909765625,2.595083923339844,0.63086,2.31974203125,50000 +3688.368086576462,7.884807109832764,56582.7109348774,71975,0,56582.7109348774,0.6106,1.7244625,10000,60482.86300563812,0.6887890625,2.768858642578125,0.63156,2.42385984375,50000 +3828.978727579117,8.191806316375732,59149.80621433258,75149,0,59149.80621433258,0.6132,1.722033203125,10000,63199.50657391548,0.73109375,2.12315185546875,0.61068,2.7683290625,50000 +3978.9714126586914,8.550678730010986,61718.4423429966,78343,0,61718.4423429966,0.6155,1.707414453125,10000,65927.25494980812,0.686171875,2.826305541992188,0.62296,2.597374375,50000 +4112.900275707245,8.836618423461914,64286.231412410736,81658,0,64286.231412410736,0.6195,1.6986482421875,10000,68637.84749555588,0.71037109375,2.5062692260742185,0.64238,2.26289203125,50000 +4262.610267877579,9.137850284576416,66861.53665137291,84818,0,66861.53665137291,0.6166,1.700314453125,10000,71371.31085371971,0.72197265625,2.449409484863281,0.61692,2.79200625,50000 +4406.237560510635,9.414802074432371,69429.64796209335,88203,0,69429.64796209335,0.6215,1.683543359375,10000,74092.30352163315,0.74865234375,2.119185791015625,0.61966,2.73540625,50000 +4543.773989677429,9.803289890289308,71996.89573025703,91363,0,71996.89573025703,0.6294,1.683933984375,10000,76805.9487221241,0.71162109375,2.6787774658203123,0.63354,2.514245,50000 +4696.995683193207,10.130064964294434,74563.37644934654,94562,0,74563.37644934654,0.6312,1.665711328125,10000,79535.18826961517,0.75099609375,2.18218994140625,0.63316,2.51823375,50000 +4838.347635269165,10.473679780960085,77130.09559607506,97834,0,77130.09559607506,0.6259,1.68273125,10000,82252.05650377274,0.74978515625,2.2532208251953123,0.64504,2.42088640625,50000 +4987.388442993164,10.837621450424194,79698.34933590889,100988,0,79698.34933590889,0.6316,1.67145859375,10000,84978.39427757263,0.769765625,2.051152801513672,0.64088,2.5134090625,50000 +5136.17781662941,11.158146619796751,82264.89665937424,104335,0,82264.89665937424,0.6326,1.6640064453125,10000,87702.76287913322,0.73486328125,2.56908935546875,0.63392,2.6882578125,50000 +5282.199355840683,11.576345443725586,84834.94648170471,107376,0,84834.94648170471,0.6349,1.67556484375,10000,90428.04349565506,0.7356640625,2.609210205078125,0.65224,2.36879171875,50000 +5425.037556171417,11.958082914352415,87402.47606682777,110729,0,87402.47606682777,0.6334,1.69276484375,10000,93147.41487288476,0.73681640625,2.6602178955078126,0.64202,2.557555,50000 +5560.306791305542,12.384655714035034,89971.41382741928,113881,0,89971.41382741928,0.6341,1.6815181640625,10000,95860.34528636932,0.78509765625,1.9999606323242187,0.6453,2.511811875,50000 +5709.154621601105,12.710777521133425,92539.17739653587,117085,0,92539.17739653587,0.6389,1.67883984375,10000,98586.26121616364,0.78712890625,2.039238128662109,0.66552,2.1756734375,50000 +5842.61648774147,13.123896360397339,95106.80141806602,120398,0,95106.80141806602,0.6362,1.70942421875,10000,101297.54602193832,0.78953125,2.175392761230469,0.63462,2.829705,50000 diff --git a/logs/self_tuning/ademamix_golden/study_1/imagenet_vit_pytorch/trial_1/events.out.tfevents.1789320798.dfd4fda67ccf.38.0.v2 b/logs/self_tuning/ademamix_golden/study_1/imagenet_vit_pytorch/trial_1/events.out.tfevents.1789320798.dfd4fda67ccf.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..de1002bcaa78ebda2bf678a27dd0d3eee1ae1a35 GIT binary patch literal 113728 zcmbTS*I^?P#DD zrKJ8htvohol&nFGQ(SGDT6Z~RIceLKPEI4_WE!=W=pj9=*)!vD@^Z3rO`R57+FCj~ z?JFJasr`5Cq0O7$%Py947LM&Lry?i2)X~yvf&EHHyZ-`CI$RhE0Wlr}-Vgj6(CkFF z>9T{?{0&f*>n$h0)OO_}OWOrb&ejh9Mb!SEA)?@}PYb=j5#ejr&5)Hnk*Jsa|Ko4f zNUfh-U%3{}wdX|Ba`EETj+Tqp{Fh@S*WLhfbX%T_()pV+e_;E$vdzvgIep}M$;kQ5mx6>^*(*`;qbpf5+|>mlI?t(3DS`pAlIG;9s3xZ>*WB6K$(eJ;ZDlJFsDI(SJxu=X z*G!CH{Rm=68fO7U&DHQS_n7LOsrCc(-|~4Iba?~}FLl>Rhh_M&ni(!!hAW-_zm%>1zX)rm|G%6o zTHQxNPE^^!l2LyTOU#_2AD+fnwYdeS9)4yQx%GLJH|4jnta15)yozVE3bS-;EmF$8< znIV7ol=^qYxd+uPU|o;B$c(1%i)D%DabLWf?wJ7R<6Z_6A@=#0XL$4qC=@I@2o;jt zy?qLMFU#2N`VrmFk0q3Tdvwf~^?atZ{2l z>vwJ6fd1FR5al#Ca@I_nT?}B(bkn2O;?PJzKAR8 z<+~wnpeWqMMDQ!pP|+jLW*#UsIxiI}nsOCm*@}?nE1Q9$Loa4DT`_EM8Y)~m6`cV^ zoBm7$zv3k-j{n+Z2MTxl)k1{~SHYQ!(?4Dg0L8&I%xJozLt+Xl#+{ob2a4iNOa#9| zVmMrRY#)yM2#SlHy@U!`u3{v6F60c~?*YXKV`em6(KDX4q9kXm*aeDTdzlD+MJrT9 z?#PP-MTW;gp+b(UP-81H^=5{FB54~lny&czCQm@oIguGnSM)x_nhU4HTfc$g*d-={U!jGHsX8OvK@p*sCR8-z zD#o!Doh5hRPJ7&h8BJGAu1>_c*y8;$02DD3nFxM`F)FT?x19irp?e<*70tN{&Wdu% zwtNk95gW>krYo+hCZWQnQIk2K7!b=u@GHzw5j^2wCs1U_mIxIsxC%q|T-+-eYygTD zTFhv=V#m5J`^#z@_C0D_ji@Xbu?}OsyBW5&R z@g(XTDx%KizXe76?@R>0!T}Z1vv&0d#lW=wLPaaCLWeyU2V`c2fI=yc8BJFht0$o1 zwVN|uFK)eKBKQ@as0h55XaI_U`zAt#0$1@_o^{t{?S`A_pcwF$8BJI8-Ou`TH!_p^ z19LI4f{EZ)c%h=;Yqbt2I(1qsRJ7(QblG!ZAYEVyihNaOG+ohZ2`n*$Ik?VB5W%&nyxr!!0NQ>hCxc8P}s#p@GAmQ zG5OQ#9iTY)!AGcQ%T;8u6&Gpm^wE{U}Ug@PRu!LNu%#c!2xW;9)4zLE90hwkCjL(&y#|f@^n2uA)MYb-frdt)&+z zcGobY>5BK|th)zQKXdY7MTNC$@!tXPE3!~Aqi>5MP;ArhA`I!oRg7WJg+iW61}Icb znbCAb;7e8;)L7K^4Jg)IFcJKUTvVL$oO=coKIZB|g%Ve>oUK^cVO<$0^yf08>59^F z*1s=~t(E@-iYCjL2!2IADuQ3<9tMSaxv@~unX8awD{S{l3%Kmig1YL^;g|<91ny%=#n)UAszg2QmLD5!) ziQre1qN3Vyqbw+lBaR3a%3OsEdoFIh-`N2a?P8eGbj9!Om#~>X+5HvU;PhlBf?rXA ziljMb9)n`Vp*W$U8&{FQo{JBUdRKvBUnDb{u6VYTb=CWR;8hOHh4UFEf?pwlUq;2) zSNem(<5{jy(VeRpEz7!I1Z`Hs?=Nz{GNb8=iPx{5&Q}nRLptUtS2b0 z{A^<=xOVsCDqPufak&4(POzeSHE#RgXu2ZFjMYFK%Ricd;*J6n!LMkBiv4a?ZlGAI z+EaKODqO_|w!(Rfeos&&>M^6~3O|Q*Tv1U!LgYb_H~5~B3W~k&mg?OE$)HfPU?TVx-B977uwf@C8nuZMD*AF2iA`D8i!m2Bd%#?zD>0+#3YS)y zI2Xr9_4oyf4k}Cpzd{8SM}~b=07dI#=Yk6{@H(?{XdQfv81i3l;si3Tw6^^r`DTP;5wJM$;84aq&18dr$2i0doFCU@p!aQWGjPxQcu1xmc;6qyq~3uJ;#H>@(L5duP{c%Vf`c626t|lEmUZ8 z6%U)R6jx%#YzM{M-OOmZVp1BbN6nM!mkA2F!%PIfVk9cAoLrRxb1}qot57k3t59XH zDEn_3mq6iioEc45^qR}Mc2DZBwHp*(<=g%{0Di?7RFp{NCWGS0sT0DGfn3GNMy$D* zai$C2Jy?348BJF_l(~y5NUaA9gG2|Q*!LOK$if1y38KB4ymK!Q)AbMOyOf_pRWQO(G0&@|4k{L}`6dJSU zqH*tpub?PUX!hR$@GBOgLe;{o5)^U@2EvdbT!o1Q>(i~9S-J!i_cfT&bj3Z_r#Kh; z4tMztik+jG2!4eXD*DAfSq_T8OS6RveXgP_TXEj*%sx=`y~T{CE8e?4L`C7+dDlU4 zD4&VoS1dzCjGo(SP^4T~CR7-36;)NN73FWZ&=eGr8O&(9Vn?5Qs0co|^)@K_-eDs6 z6}G7GmJQep3jM|&LWLn$;Zea-X!%WR3yR?_n9+2_cl!cVFavp?Oj&NJ~}NnxSPXu6{C73*%Z%fmB9oW>Z0t@gYxm z9K*PZyg#fJwQg$9<*=e2RWhULiqZ8tTnBb~M z`Bhp%IZIIzHfTR6yk|0_>56ZiU!bDc#0sB}T42dU@GCZHCR!R)h&gJ5!|_0 z{GFwk{bctsP>hpn_1|c^!l#mTSM6S(iE1zxi`p>}{E988aO<)VpF|BcQx+bF30HCR z8%xopYw}D`?3&ArrYmmdzrwjl-)n{|%5WJI!LQhciU6zSk6^(~E z-t$wSaN59(rYlbLd54Pr{T&oxE57m~g*X?Z#=OOE?K{q5BKQ@)s8Gq=Hysq$9|s5(rd-8swqn;7vvN=zc+HHaE1C{v zttdP1^m(wNv_CQt{EA(uuw6fL6eu3|ju0xwa1}SduvS!r%@aRR#Aq_3>5BGhZ*ea6 z#vJwsg_<4{!LRT`h0WGtyuYk(g<58jpm=y!&%YcpOu>io*LWg~#tp zPM}!b?aBawoN7EA=c;y5aFZ>+|D>wQmsCOnR*T!mfno6*&p_QDL`iTM{Tv-DV>A6_Kcz*=``-8+;h^O{kd0RRrB-DT-c%YkI>DCRY*qf~7d0H{BCf)Vq$%Xu6`266^a5r-yov zL6O{piQrenqr&n;IsQt0-qi&{#VoF3UM@@F7C0ye6t+3cXu3k>RRzvP!5JgGUZ^}^ zBKQ?&P@!8~mI^ECXfp?)Vm4RtoPE8J-v8(VDDG%7qv?t#RjdXQU0A9Bb5T8piQrcx zq9Wi=3jV%(sp%%6Vh&f~!d_8()`j6yUh8Ktqv?uNFV;PfUCCpwfx^&+iQrcxp`!Z@ z7kuK*G&@+Rn9EhjU1F^${h0W!Fc&v!n9+2_;NU;FqI7qTdI<_MgOL9YfM1b}imF3K zTVX|wdYmc@na5S6us;`TMyNjp#neJ(G+j|@{t^|{cT%T;!nuTr;8&!eVpNaVp`b{4 z`b?;p&s7}GVa>&u`zwEd;@dlBG+nW1WECnlMt<)Lij;3m1ivB`6+hn%Ed+(O$tR&= z0atM%m8F=|C`J<$N)wpTbcLmR2`c)w?10~hX-s7z_!XB>acxiB15ga@RU=d^oD2Q9J|hGT#FDFs&SuTU z_Sj7qVJ@bpGNb8==QTAr7b{$|@z+&qSxf}K;tDG42Diq4U#z}4UZ_~aRs2q3DQ>Px zRshB2N6cutqFjx2)ob(WPiL45^VduSzaj$_t)k`fVMWc5T_jX2<|%d&$t|J`UWh~$LR|Wu@CAtJ`>b*SB!luo3Wt~H5i)p{ z55C4!?we>h9rA+>`E+`66M)S6CCcE17Yg%>5%X28MzUh@)aPfVnrFekhIwdX$j{^@lHq8y_rN@>`ebGP~2eGhx3oXx487iu^CP|9ln z(iB*qJ=-Ow)v&_Xxrm0-_e#!X_=ruOG@KKUO`;6mb0Xo6=VYDq^4kFUuv%lQ0OC*i z-S!;y!}u}#*wf`XK$dP04W~mmmtoz6*%AOL-!97Fg@pPdiztH^VjO~ynUa^i0OGN2zOexEoAUck z75^}V+?+4f4Irw)qTzH1r%$}iAMzX^6QV>JypWft5Rwq0mkE%Lj@{w~kW$JI@;dvU zM99;EeMtb(UMm_-hm^7V#HF>)-vJV|MU=q{@w|YL;HfY5021?LaO;b;ORI?T3#t(N z0|@E)$+#~-roI&or$abPYyGd5o&d2Z6=m>3x@9BeaN|Rd0J8q-a7zIspYoFz%M#W< z9=^1$N(G43ZP9Q#FwoaO`3?Yw18N86RCkRk&(&}n> zniu!tab-UkxCa{Ek5W<6KQ+8|9i~Jh;jrgiPP{r znzCNf9k_V(ZD@F(WzCp1(7=Xgdv|RsXm}G9L|NR1H_kN_ zpmcOM;%9G4+G$N#;k|B&hST>-PQx>jo7M=w$q9j}RoiwXHA(dPv~+^Z05W%Gj~4=nALZH0 z$J$8L3+b7pm zjN00>-ZFRS;TwDsW4Wy;h^H8}Y`&azUZR)Jfnqe?RTNTRF*Sk!Pl+-bSd0eP6uyIEq?j(s;ufQYQQvUSmI~ke4%XwB8=~=b z6r%&aviJ$r@!kFqiqY14q9AdL5$Eul-5%8%ijm$^QEL6g=y9YY7NeAw-+Mzb8t_(> zCsr|v7-@)q?Wyz3q%7DMV!wzYdH03RJ@E9&2W+*6V&qdPN@-XznmWC+1MHQX6_tzy zTg(>9^SJI>KDatNK6tGFEBuJEXgGbZq72@1(neGmjIGS zc{1kn&+!O3FQK&@meyVu(QrD1Q&s!C9@`TjDjP)^ybyB}g!C&@wE~ECf`ZYN+NG6c z&3zza(%Oj#>E|+UBtQmdiiXo6oEb4I3Rnb??~gxJRQ$V(z`1?q2Vc-i9+gYc%Lp@!S>^P#=ILeyb*In zX>~O`t3S?oTzd>UT0_G-y68Rh6&mz_fUNaS%<-tcHai1th zu#z|p&v=s_>KxRfIu;H~jRdO2|-iJg{7PsMjmt;4KPG7=dJ+8_UjjyBO zotM+Wg`M9;<`pzNpJ$>VaT^}z@ZRNG?uCXou0)htf5Ur_;EfG$Sp5AmXn5ulrp(7r ztcIuPb`%?4tJE*kVP6PQ6h-px3(iyU^u^5>BM%L4Xm3$U!y4X?HoZz<)9<~qy#LkO zO+Vg_yI0ORJ_1)~8{NkPV1+MRBN|TMD>=nznd!t;a83?w5@qn7leF1*P69I5NdRQj zkA6D@kcd@W$aUQ%2q{}$u@WHnB-EMVbO`50i&jB%{Fb(}oG61A;JK1ALV65JxdIT4lkO=3 zi03*kq}#Y@2zhFkKOZ3438LY22xmqrZm0|d$n1-v3|@$YJ3>s`CF#gxg(WJe^X?{#LNpqJ=l%FKsV*N`&xQiapUXVyEwZ=ns_> zBsK~xhql!DSJF$g8{wX;6#siXoZI+WL=sOsIyl1=&rP;k^JDP$sP|YBDfQK(Df?M( zh-P}c^odxZkf|kVPVGk69IL!q%{Bem}GEu+W^zQhmvyjSFr4bU?NG}n#4IkS+ghe zhX0L~ICX=?a158J z>5NbKTKnte#CZDd`I)_Yp3i?|59g)gHxa^nUUFII#l!L7Qm`mK`yrQEdnG9FqF6$s zSRc6^dl!s?D}nYWVm#f#xu~V&-nKs$UmVTS+Js{%`gAH}kv70Yri z!2*U1Q1q6w9>)mSyCw zmP=q>^2>M%(fF18m7C#h!DuCa(ib+*8^x25rd6P7*8 z6k6b$G9rnm1@=^B-6K17UUojTz|((-l=@oWgYG}^CfS?R{%xQI%FCFEcy+bFvsYPn zHJ=?T`UNd;avLJ*pIV^9)lU1M1=@5WBKce3ueqO)SvhveAXrgpDnyWAMR8hSx~38? z%P(hhk3b7tZA64Lumx)8UVRNM@cuX=jN1a&=&}CIsOr?t4i@H-Swuh`E%4G-)}?y? z={}Co0*6}@N#eFZ&H-8TdNyjF--8%Ww{X5H$==f&o0Fyv5yER}6@iwNYj4MaWv8Y5q#LzYg5M`7mJe$p z(DH6fDE=`va|dEP-SUOq&KtdcDg*Npx|0auwN#uy%jnZ8YG85qH8?D=l%AqkJb$wG zdd1Ou4#2#q93aNiE!-=#h08*)xP}rTyq1@z&@$!6RD$99TxrBgWG$oEEt9{!Iz6v|mDm@LICs(DJj#uWw*EZPRM0z>=RpvAi-@M$6a{ zyT5?t{aRwYn3jlN-{Zm3+nWgCwZtc&<;4rlu5f|cQr_&Yz;ZdM&gQggo&q+fEkTcJ zpgCFB5J^1EsZFLKo|}lqD(#>-^=jH&bUgJnr#Ui`cu@T#3iF{kWw#*W>S|8!wMXHR z4N~uX6Pi=M_C(Y_HK(v4SA(HB-R(+5@;9fU#jKZ$?NsWf3@d6+Zz4#rqBzZ|=uK+` z9vyLP4m77F8bn9~o0D79Jv*Q|xep@3xXo!s-<`NHBbF_H01NYhArVkVbGnfogh?kQ zzaNC=w9$k}61O>V4p1XuvIjIL|8YcY{mrSZOgc6v_Yq+?p*fwkAOgi|PKy0lUjvn0 zwI2by#Om2ZChsn>J_%1@)tQ_p(43ksB4QfWoW3|7nGd_C{08lJf*mL49A)<$(6%2g z(>p_Y&w;f*YX>o2%-wUwyS-9yUT*9qLU_+h;5j@m8zhZ(f$jARi$X%dveuLsPq%O$BTkx8(itqLClevO7S9W4sZ_c#5iE`Dw}cBUiI*vs<>@_f zUi`Da8-b<68e%-%!f6|w+{ZTtON&iJ2(M+?WtJu5vl3WN?6*+KshyX|s}zgH{*GuF zfBp~N+n5$gjHg>T?fgWGs_9_qc#;U=wU}Q;%LT)07GSB=SZ^h;gj}as)VkA-op7>u8b4N^u0smseVM1s1<-ilyYW99ndicew|a zw;ze|bPJ~izH#`8?fgzT5yETfmW`IcPW>vta=>rUz+1IzY|9;r#p^*^wEWCcSqK)r zaAG{&!uh=1E-=3YYi!OLB81l>a|bQjHEwxuu`M}futs3{ky~dA>}%Kw3s;LxW5S>X z?vE#ucv_&2xfY(AMs`7}&;qrRiIn8TD@yed5hPeqoEEtDt1Ro(57ml> z7AX6I2x(vooHfV(7PLU!w?r7X1s3Rr;liBXH@Gn@%x<5EfI3>Ba&y){+I&Kqje-_f z@Qp|kw*_(zFk1coU1))YRYYw4EwG>ReQbdRw%5->3p9~36nzxMYJpY?V{lH*EYgZ$ zmv|*dWb*D3CAl~mkLH?HLkl!)O~f>;1?GNgWCgpY|KrFg!H!dLkFtA)8t+1qZ%u#v zQ)~T;iShK^vxI%0#&>v{HJq0gcI}m*v9#exaLK! z088~^V!W7^1)ZGFf~C0w5yESUe2$jGxsBF=CE4$@>7Cj&mi3xqIqk`MBZa!bl~l0! z9U;ckEu3#2-z{#n04&n6Li`!eYJbQlUG+dzCCSFs?t(}*g_jNX>h}OaH$h-Oo4G#F zoXkEFAq{L!XO8y12+e7B84<>9PT?cZ;=)vFu{{kIre7ryP)BpxYmMb^>;)pIAp*hWONd$`3oN{llo<3fAC&mYM ziRSHzOx|50@I9VFncgKgp*ij9O2jm*IVm1(-v)Niy((uk1v^gs2g>g09%7EmH1uxf z30Uimb&2s}?w+^Kd>jMkW$bt&g!jBG{D9|0QTf42uymSK>MO8Be5P2&TxT_>Fz=uZ zVEH(M7*Drw+D6bL`Ey|DW=VwbTE=`vi(F&lL0~c6vq180?Uf*~jAEJ4-4o|!P}Alu z!O|v}7%!&f$mxkoz;Y#=2;sG8m7%4VUEV^l$Uj)VSYYw}L9w{aY>JlV7uC;#WqLU= zo^IiMBb*m6VF8xm%~yyHg4fdF2U-df+{?jo%5~ydfyJ|mVhNupjg~Ak=XGG|oJ5SL zTR3Yh`?=Ffu%uKFA-ooeDzx+~F){#4+cW+K_iEQz`5%hq_)%4~ybGQC1}s}riSc4u zj+%TP2A1{Lh!9>&@gKBwcFawhjlN1(x+D;_7OFquAfx%ujA91ud|j zCK2^dEpW}Gu#a&6Y_Kj7$=?E-Ny;O$V|+FK0k+jfM37)baay2zVIw^C&2&r#LJQ0p zNrW`81wPW5A_Fb(>v$rJ+X5R;zJVXItO@;dU}4_0AOh-Wfu_-qF{xbhi9NKy)LBH5 zxGj)#fCqdv7eNbrwUCIdzXiUKs>BwUG@)=Rw7|e+M4(tLaLSNXIHzagdg8x(2Rjg% zyt_nEBito=>b=7IXZo&0Ov75>9mOGdg*Mb)QzO`M3bs;q&)+tzD|D-Gp>1II95Iv_ zPv1Q`-wJg&bm|@Kp2NoxA-w0MKoZZ3W4=TxTnSbjaawx6_Ntb zJl(>%w_*I;ZxdM7-ylMGEg4PFqGG971{TSOU5Dn?&P(zhip45o3(ia7=P~$PVBia4 zyqFgCj5CkHGNYIX;k6{opk?-@QT|}L;Q4Evz!Dumv1oMtf!BtxM9&DA7s(W2yqFfL zUH6}XCHDal!fT0^LyOc)_by-wQhiY@u!IItEdAQCo@C05zIF*LYJ-XKbPMMfNV6JO zt^mt|(L@NZCA1k@!s9=9fn~Z?^|%MMYs~);#WH#PC!Cjr+>xeWNp>Q}(=D9OOJu92 z_>_#{b|Qq=;@<)-18NNMk)>n#Yn%j@%m23sFSd@BHujp9ix;nUv|PMqgq%#H))GCW zB^LHWRr0-{LC^wQ{2-EeTA;Q>bNu|ReIM+8^|!#R_Koo+bb5{ara%jvb(9Dc zs|6M$Y{o@v&}i8s*zs0H6Pdib#AVhlQRcI04BQLsmQ2JntOf4v{N4z5PnQ(~9}0FH zuQ1B)Icw){T&DNNuQG?-bIodEyqLRZn>+eOa9(DwCqj77i&rb$J)7-odIBuhvcE2R zSbHV-9YL`;Y#)x(Q)IqpCs-sO5aa0<&V8B(eTL$*blaa3A-tBgttbGJIXYP_XR(bfev) z+IcC8rdZVVS0=aIVlAM|w2_OOXN*!fUZoM9Ub3t*(Nv856Kc( zauO*P#qeVM?CWne$KN}bR1)Lq7S26dWu?mNFfTu3^+gB4YnjjqE#FdloQDh4D(N|0 z^J~}Gj-)!9laX3)RGpra+6J1Fi4u{-)0}cTvwm~j<>XLZXig7$5h?XGC+}Tt@t}5k zHy#VkX}%T_S66fDY|LvB1o{JIL+yRLj{JkHF_}(nv?AuBBX)MDdKD1D`-wztcfsgb1GW#6`PaHvA8y{ zFz>A-0_tc^ib|}T96PtXxd_c^qYIHFZgb)s;Cc6XAE7x_ZYE;uZ%)N2EwDLVobr7c zG^cVOB2cX6B&oUw7wx7m1><0sxDZHW^6nC{O1Mkxt?n%e&1qs75!0~dl=9g780?<; zYGKxb9VacBvU?T{_e0W)o?6(PcIy!1>ANTA+lRX&CGotBHzGoK&x>(qJTE3&cJ~C! zyyuJ02rP-I6ietO*5iTp1|`kmb9;X%pE)l|OQSF8nHUA(_u=xMD z;VH0$WKb;4WL~3XpM5W1m={yYo1*b_3#ZXeC^BmTmaJw(2(P7c54320FvPcbC}b(; z3M_tE6w694)^CKv%FmXA<>*~vJl(=+PEB-I$bqHNQzC@d(nPC=#RgEQ2t&xg&>0=}M&3*8&61wZVhZ2)>pBEig-! zh^wmw-g48yS!>t$fHd4c3(+Q`{;36O*`~jT7Pxu{5y{^Ir|f6_4tQRdi`!sDm5m^R z1S^Wu0(XBb$8&XSxkDVZK)vxqNCR7-#<~0WE;sLKL>RXPdS7C7w>j<;&0t}kno9)K z(E|6Rux@w1+GKMNTA-2@ktA*l3Sj4w`4#N9j-_}oT3C&tq)oU2;t>&<<@Vp>du@LFE>LyPpsgkrF?(_8RN zV99$*vApTdx_=g;e(Wk(;)WCB#kAzdgbxABhVeuQuO+WPTE2e&jL!vr-lw>zpmvRA zy`WeYEE$0FqI7FL{VKaDUnuCNqewF1ry`x7S3nCD1GE^u=I%_LU=9l1JUx`B?o^dXTPk*R$$rtzRu

?}nv=^8BI=)-Q|yWxC!skV@FODmn^VgWRzWNuHnJkIL?!pn=HImI3!LK@hdBGwKIf%|8f$B8g*a|*k~>O0E$SudeENyicabu_2I zY3*@|JFdtLh2~V2NF<5doHz%VW>Z-L&1p+25nF$AYIj>1o0Dx-`*qNq3^Is7v6|D` zA1Ck<;N~}cHtZ5za)?acU1F~ep29}UZvBDgwBkMy)3D~0B^!%>B`ki%%0q%3C*mVz z_w4rP5q_lCK2ZA!Yu(L~7*F3lIp02PYbkFB=OuGF5yE?3A_n1kQSGdZ-*9ycJ|+FU z_DT@=g<=U!X5A04T-)*pSZ>7-g<;%tPLulG((fq6N%-r|_RQd~{3 z_&u+}HMVf4CqCn-;6se3TR5MWk#}1gfTcqK5yER(HViF$+PYc61?u~uyml{Z=jHk) zsk)xoGP&etz)l18vIVoNtqW-BlDI8qZ7@E_t8$=|3bBgQ5`hwMEd8Z=yqz}m@ zf&?py)0`?Uu$q%?hlw)KobnzKAq{L!!P8Q{LvvDiL4Yc36UgjbK)G}_c51yL30ZDM#R?NoK`jMi_NL!$~}9bIr;x0 z0>x@hNrMvc5->ICF+QKrOX`8>HQ!M9eZsArtHFr2#`fVk~i)qo^UW4BVH{DN!@LEQWLd(K~>Jnh_JZC@Q zRqa(Rc^AbZweC4uIu2cT2G-b;bYeW+!fAnt`<*_+8XJ0(2;sHpnW81PV(1;PRFA5P z7g(bAQY=X0?W`*V7wHI>7iONA9k(iA|gnz zqBt#Zcogda5=U*>YtRDAJ`y1fY=J}Vn+<~&_`Qq>&0&sdL;%+#(N3JY^!6%kNJ z3;Y!Es%47NA8M+De4*T1qX@qM|!;F<2xrm4_77 z&P&lrisj>0D_mo#kIxMNOMV_Po^IjXb84^p8=rPR`-%wRwG>T7%ft>fFTrxJ+ABj~ z$&aB}jt_f^YwSnmss&(CX|_W&o^IjH%aw61WiT(zI}jnfmi*~xc{Y2+X0S|aXWryZ z?Y!iirdWbMq@hK(aL8z|Z0$jer&~Dp&L5O#;V)gH$7(3~cz5nm}$ zmte<9J4@L;E9T8blESR6O0d=q(~0r)-IH@wGckRES2gdOL z+Wytpx3yQb$cq%qZ>2kEsg!fVZ-mo&665I>&K3IUpgr5b5~xLl@LD40qvd>sUM|eb zvL@4S3oIe&6w8Xk`nbjnMoMAZuva9;(=FfGZzs<8Ny2YYPpJ?gyq1uKXz4e~+!`!9 zJo>kIS356$*C>{$S8n6HoG)!N9@bc<7cri0;jFPLlat3_UM>U?A-op9MQEAo;5r^G z3-|c15m>g|pjb3_mZ9bOBbS?C84^s47t^w*SP}p7QsW3Bgx9jg3N0oXX|2Jsr;p`x zfu$mcV##UGdV=DHTxTMASdE!1?{1ET9Fp>`g@Sx4;K(9r4q#+2x`G zd^*PVCxQekiqis^9> z!i+K@0_tdi@_DSUx^8U$;SKlCJjW49;;D3zws%yQ?Jr z752+X^K{?WUJ3H@DVANkd*V`4o~iByme8xjc)Eo%FE$ZPOTaSwArZoBnQDucvr@*1 zFfUFvI^hCKRsqE#@t!p=LrxFP1j_&?V!W7^_v3G_1IsH9B81m6#vUyS#<5CZ=`^UB z{D<0kNqtGN+&#nkK~|f6YH48c7)y+&TR8t}irX*O9V}UMh!9?jp#xe5SoE0*mR3VM z%oJGSizpVEcP(&@EgRGr|5Dk?@x*w#g|o(bt~=iYEOs-95MGOxBU;+JcYXsGs4H4g z=LD9Zw{Dnp4+=Plus7h0i7;`I}S6*By|#rpU1Z zJ{^ZGi6Fs>;xwm14OVmVs(4xg&8d?O5z@fsrcMbO(45X}A!6%qPW`4$#O749choOv zPP=`HK(U%rYpWNmMe7!Xe_?2yKat71OLTO?Q|NKdz9%#%$q*u@Va-WuLy`gPo)XjL zzX^7nh+@j_X|RWNKcLC)qZ44Q4_Zi!r|+JeZy(yYkG%=I=htOK2=94OamMqKYS8#S zSZ3Ld9{#cRsuoy6u~@mWzTt}f^%1WGHEW3RbPMM`jl1d*9k7IKBSLsB9bC}z!TU76 z&uP`0zOe#}Zz;tRl*;<^BhU8E!ndskd?UuwEu1U#o`k4uund+^6CDJvrI{OAPPI_q z4)fAoYNtX;?YwxFQ!KsToX7oi*E=HxSYyu3iScv`XN|qu>^cK1ikd_SuSLQgExD_X zVB63uI5uBkDgQ~a#N1%r*PM6hAU?VO<1;ayZsFY5RF%z)2FrwBLe4)SMWfXlQ{CJ`hp=)B@jBc|<@9oL)vm^0z=+H`d=i zRUX�j#LvN+L+GqBt#Za+6m$fh|rv3xpPE-pE+A6dKq9H}8Jc65P7RWikNqeKpp#?tgO2pRR0!@ZY z#}+tQF+B)cpj00sP^=cXGev)crh&sF>2;uY1^9! z;kCTph;y92uZt8|o=!?zDzGH`P%Kjyu>RV&gU!pwFfRjU5##9=PNQvXR;US<;fsh6 zUQ59yv{-aIR|=Nf5@sa=OZ0AvMc=gqx59*{(s(a$Pjg~C-NI=b*G(=B1xsOjB81oS za5GxIU8#%(i^jFX+FxqdSm-{ArF#+U;h3rm*_tpf3+@o(=@!oCMSk6BS+IEJ6Cu2o z+%0I)O}L~6mJhRM1qm$v2PhWZsFnD6>9jL*J6MXB6XWR?PCH+`p>Q%-dbtuIyq23= z(bCN)`~qB{O54toD6O5BV?lK`r$>caxY{<V*fz8R|{qaT6oPwQ* zFm7`ysbM{QJ8YtwJuFOHcOsyU=F~jg5|_Ada9#^&PM1B2BypP)=Kxn`=PE&Sdb*v6 zt-m?R_|C)T^xX6MV`xt6cN2kPHK&CSzv7~`KGJd_>=Gjm5Se_tgr~(eJcWxhbmE{n z=^Z9w8rGb~Y_4h#yXQgqKK%tdj#n^c_ndn4Ixf>G3nco(T7TDx7*F3lIaf6)L&+34 zFRFcs5We#wk-<7IPAV_)x2PZXh6e~NzYkL^Nmp5KkKEtxh9Ov*gc9TF7S5Gmh|#fY zUVLU=8gw&NU6%XTjxxO@nTxONj0(s zi{3;cgx8Yng_hWwIb*?MteCz~U@1CIv0RzNx(eBRn|K!HMY=mNo^Ii^jlf11@VP*_ z{zM3`<;)JWjHl;|EbRfplEu7EGV0W7en3rwc zi4a~(v^QE>9xB7PtsZ)~$e^rtjpf8rEQUSP(Gr-s_9Ix*-xA~L7S25)0}?kXyfPa%@?)t zaL*~ii%6-jIem;_HK*xmEtH`-Wd#v&bv380-IVaiRw=B?h34dNhKTy7=5$us(+rx^ z+Z#kAe{&jR&bqahz4}-ptf)JWi6Fs>;xs3%3kA3=zZ`IR1LUtPT+wVl*}Qw zV3*iAipb>KB|P`};$#f%r}`3_(`E}IreV#g{kTS9a3z@XzDJZ`$4UGDj>CELg-%IR z7HQMCwy@Uo%ZTyx-ILQcDu)mEh4ZrE7ZJjDUL-%=QQ*DG<*y0%xOdjuf=~CTCOJA_XCTz$?6vZOUNaPMZtO}+60fhkPhi=SNwHi_W4+P$ zkg_MuiBPR1I)PQ%aXi zh8Fm92$96o0+U~{_V~Lu_P>V~=sST(sjmfYo5rd~{kOH=3oUT{bRw><7C1?r)lVHa zdqhDC{A5W){Zk86bbBxnTA;fv5y{^IW2#upf-Bzr;FG?|i3k#`C{7F9zk<~QN4Kq- z1T8S%lL%>G3pD#MYb~_Eu3khKw*}sAz7!Yc)S~zRSePUHiGVuppKW@{%Gl_lJW*D2j`@F z2rPLID3)~@sc4xqFnT#y?kp$9(=D8Kek$WV{!Y-=jR@hjEDu17)}~RiU{vmN&aXe{#}P>bBXbE3#SE2{CQmo^Ww3B2;sF@?MKUopZi0=qHKEB{AcYN zOMOPMbZB=M=Vjdfjf-Gjnol9di)k4aBXI&OEf)|Wyq5U~(4zixbR<|3N(}u4miSi` zOW(n)&%Wh`kw?JN;wLd)Ov~Fzrj5a}O~OcY5WJR|foR#@XJaZ{pxitbR|+iKi|TGp zk*qggRNOWG3C$^RJCVfGoU~u-V5_WjlNkfeDbAlrsjoRra#zM>({0^qynm*An24*Z zIW00^-HkrKrQ>^OPS&T0sDEls<0BT`gXYx#ED_1yoR;ljz2Kwek>=RNJ<^FF!HVKE zr`^6!@LX;E_@Waur*+vxNCTVGcirJ?(44d$5MkWrv`TRmF3hmO>EW<2yS^d<>S#`3 zS6L6%bjVHJ3(ZOQ1CbD7fq0XI@~GLPhPB zAmR;W_pIEv40nk8dPk?i?s-3q7*F3lIag?RSDD*zUgD#O5We#wVZk~tE7vK10LuxL z(2W91;CqT?Q(gvs?UkT0Gu8bA@?He@l#~TR1IHdDbm?u;hFpLU=8sgIN}<_1D3oH{j&d%Gx#N z`ITa6t9=eFTRP<7^JgCoi1Bm_r#ZPMul@y=;iHHUUW;)ET2v~Btp>~Y%_@NcOL;lP zGV-@G>+>=x)(Pe%+K(7dw{Si$Kg`$ZgC*-A5yETHKZF*$UmH(@#W*~xMqnwfpjdK$ zu-+6C-1Rd4<)wBp#CW=ebHDlC2&Y)EtW6?9cr7}I(ULcDJ${R76P32Cs&&b*vZ`8g9Ta!=Al2f=I6ID(e;U$)@Sl(O!mb#ya ztE&YLxzD-pY&@nM37)b zaa!OcJJu@&B&<(8fEGC7ED_Sc7N~T(^$uu(eyKzlw*?w>W&PBrv+*fMhlF;#+_(^0V6U4*lDI99bAWv|>!d*oRJlvU*53kkwtHd=)VuvS30k24Vd%)u2>u2|? z_DYbkg<{#%?krl?Nj234%bXx$Jl(=+8_Jd^@tc(SM~M(#OOG(LwCjAP4_K!C7?CKj zBzsXTOLSw^w{Wg%YRgCCyWPJl5Fxyl&PUPGeBZN`U`c(qzU%MW zd5QL=SUwv?p=Dyg;1w`0OR9+RVp`f5J;Hy14Y7w9`X@ z#owP|(R$7LmqwX}*#(#vg;m6Ox`ngG1_rgnZ&BUci4b0k+%dFRU25$C7pQXkCFB0o zuCaUj8`t%l$AS0taJ5B;5A6ia$!G(S#M7L*U1h!b!nd{K7ido9+lZ9gS){Uhn z#?9=YIjz}4#MRZD4B}aTv$D-MdnIU2aY01XKQ$+RiI-iVIVpw_k^IeR!($e6mdz@B z_G8#7B1o{JIL+x@IIB5L@9{eUnp4JEBBX)M>EQl@3!ym~rx9V?=5%d8>t{>FD?9dr zg_(Yx2&khuMOw2Sv24CY)eV}H?Oh^C+~&kNz|4)VJ)t>;Jtbo6Z%(guwqkRt(N;eR z%_;aT5hzx3I;zteo0CJz5N+7;EWQ$%yt~9n)-KWLrOA6}PA`5DF%4@@k)btB;7agK z|GAxD$MFiJ?4FNavz`q~^4Qx5cF)|=#CZDd$!Qzo)U+(%yj(ITLU_-M^l{uhJGDE2 zS7^_}7cvBv-@z2i^%~aSnVV|%34g0R_#`o&ZsA-By2m}nx8UX^5FxylYL>;m=awj# z7w@=io$A^vLFo~S<(2a|yh59tK8kO_)k!7B(=D8N*;;6Xe?`hPiwNPhRGh$hQQkNJ z?>SAm=^7}o6opeP(VAcIS~g(qnJQRglO7P`=@!ns$n>wlzl%EW1rfq)DT_qQW~pim zn3uu3yo&^u{1X(*^-9*iG*0grW(=0&fy8(*E!B$I67YFR2_r&yEhQ(>V(acX87v0Z zO_$ZwuCbg`6w9+h);EvCtP48I#0^i0@pQ{i_S=b*wUjNvqV|Re;kCRxg_iQrQG?-PTcsLP zCb0ONuCoQko*0Io9PJ)M6QBid9!Mndw7`p@tmn^e9k`zhEl|RMNU5&{${l6hSSqS4 zlz&s&6IJeF4 z>4=|61PN9Yrv=Vh&w8P;(daKy&;t7}Btjb40;m2AFn|_VVnc*+Tj2eJte=zRYaf0B z3$vpO5l}}9?32WL^tI1@>ojPAs-8rWxGj)#fP)J>PeKcH@g`#HZ-MVl@4yy#SND!A zv_R9nM4(tLP$pavTj2VbrarJsoH;^d^6e6yqoQ$_$bMBh4qD*(6GTkITHuR|Tk(BP z-}~ihN<{ttN|2a9**&!{F2>K{%S1UJSnJ#Li1GB@^9OtP{9|eq4ClpYBoV@QUL>Bd z&daP*zwxiIUsKi&6<8wAQ7n3Xp}1!TC=SD45A_*9jHg>TZKIh-27bepU_ylOTJmCW zj?4S5>I7G{;|rgB5m-VlP%O6^vF-)la4}MYc{$=pjHg>TEinA?{W&l%G2TQ7ujO_u zT2}4+d=D(LW{%E{YS)7 zmeK>|yTP(M>U@&GvgInpl6Q#pkNY)UPyGhVo3q4tx`lK9Y>&)H{0naLt`H%-mMigS z$qvuKdAWJ)N*Br6d8x>xSPVb1o}f^PZNaATu?N{}~eX;#Q$urQLrdkjw z^);s$%^rAAa!Z0ILUX#hh={AJIa$4DeJ5)EwEcN#PLFJfsDEls-`ut@gyuBIjfmuL zPIJ4lex_=6*~A`JRLC|WNU)+f&8fE!t2upto~;VaX^lS-(!l1lVY=TXXiftoh%jz* zayrcVX`J!zn}x72r<^7N>S#`tlB`Ex#|Ml`g66d3GLa;1bK)G}8_mv!(42H{6S4I- zC++H;*qp{5-K_}Csi=Sm6stLne$pA6lX*xp{ENeJB}69QF5$WA4DJ%U9%QtD=5*|T zot=qWPFov?qeCPiITVp0q-Y?7MAQihDTOrXh|Dq_Imf9~$`O4jqELvU2$iJBP$EO7 zB=c0HK?#-UJMaFk@ACZiUc27>0nhK+_gZT|>wfpUhZ+*oG;?~A(kz;{6P$?QI!?(o zM)!=3;N7=TcXH)?SnJ(RlX`ad6ztH|rTes?dwOS*5V3KQN#>2q^ih-WI?Wd&rEap4 zo#6RR21|({@81H)pheNZGSZ6FOUbglYJvf<^l>2}qAUrgaE>n&m`#Lvu~VINh-0~T zhr#mr0dHQs^8)$P} zwL=1K8&g%nX8?=GD^ky95!l9s%iZwp#8*C%5K)$cNyuVp-}WzHnY#6!-7k_gmR7)E z*;lm}St<_McYbQ+!w^Acf6>ZN2cPMojWRPbAK?=hNr)&*a0;?$TFRQjW?L)oGLB<;`7g`=Hd}XR zx1~DHE-wGRF2QBp|6i71K6VVwkXIWIW3a&Pok^021!Le}>eiEGFL&kf+c)3!xk+%&Mj$YCVvrz~(}{TNfQ!0E;$QrrTk@8DUW z&*!K3Hcd}662z@2fd$G9y^j<4+oJ2;!2%U6Nk|hdP%GXU??3BnL&Ag>m=MK#NV@ua zb}lST6MGWSfCYxx@@`3%DLC{FEb#kWk|eDK3Jx&tP~>8;K%0dmws8ymawQloaN>a8 zy}$zJxsyPtEbzQ?FSI~CC9jdt@svGCrf8Qq!Rr#fuWZV|0@r$xn5J3as-=_gOUyxE z?!^k9v0Z6ztG0XNTfWkSk9@M8{>z8SI{Fr_P(e zPLR~8eJ#hbrI^7IR&y9nxtr#_@xDRQ6W$`?VEC;^I;C+Dx z>-OYuEL9Z@mJuEHA|u*sewb?Y*~&a znA01164#J9-7Hr{r@QsQt`N*=ct;ZTQ|9!Wl~X;K(;8J0DQ-?*N2nuazr=&4u%Zt4 zBthJY5|~rE3-9__ZB9QAFehD264FF-Qr@Mt2(F)v(j{R+b5e-rUG-kU`|I&No?chl&!xL%_(`-Z-c>{Hccjh zQkj!pNIx{Eds->+&?W3`NTz6)=yV>t#FKZqx4@i^+mo24nbVSg9}|R~ASYGvmsXOU zAom-idsa^6-A~g~rvl%wFfNDGv%9C@cjqhdWmbX*qj#wG34VjW;PIIMId z$8y=LrSK^P{Tsn}-nQ8J5Wh9+KY`S97%U@Xcwdn+9C%b0SUQa)^-{7NS20P0d1*JEgov`txQHye%HF4eC8>UU7RM49 z$Y7}+$@@n*#Y7k1fA&}@8lDd<^T!`jkeAF$ z$bT3tyJdKH?mx{n#%H&~rjU9zi@*ZSRl8UMOMx8;5oMWp8Cg1*XnuhWs`m7t5RRoT zq-8_bob20~;+aztIP^A{Q`?IqNyMCfHIqd@**>K8Y%r$_S4m1^=9Ju?_spYR%>Xwr zC*|8Dt|4x z;#QQvoMu<@t~qTosHp~XvMMJbO*E(R3hy_7IR$+tVM22n7|Xj-bW;9s04&UZzL9_i z%t^n7ciZ`hsr@g4IhD)m{8-}Bnv>uFuY9)b0p_IFhQu~*PTd}bqd65EH0TcIq}Y)J zN@Y%kt^?7WF4U`>7<_BJq5N=HgF9Z<(y0sB04VPdE-)*vTO~oG{2u-^S|yH z#$tJ7h273SX#-xTk#juvqn^zou)qt>FY%d2^Q9z2lx1`l&hfH{cw=CBx50aI8_BNr zA%ekTzHmIUTs1LD0oxe1h19cI1oINn%1i;)t5+xq5oH;E1zD`C+7tpyMqh_(9LwuS z2Frp=1CV9Uk&V59<%2G%XR`<_aMZPjIlxk6NJ2ze^s|x0b3!scBV6>dP*FiLFZl-< zEN7PTerbGhFnS3r@0%6@?=@s8c#(7wSXvC(_L^hKh-0wW zcjkQ&>wLQ}xM$BYCG~6;!5Z6Z7+C(D7EwE!H&jP!sDQ3Xgp%P4jxD_R^ zzyaUy;RM>so_h)wIAb3PX`%&M%OAw|pQ#-oVL}UB8OOWoRkgrQ0T!mwX%f(Y1r+O1uU{wZ5lGXwR2Y669-v}(w>?VnA+yWPSMWO|6tY4`O7TEkD36#nLGdJm@ z1x9CCbb>C?s)%HYbqUWaye=`l*vttma90J1X_^IUlm)3n_smgNpT%{Y!~{n7{Hzd; z%XG~2PuM-*cPI7i?kU&_miq?aKeEHMNr>3E$f#b&?wQiPZ98B|_FZ1VvBV@YST-BK zL6-Y^%eTRKo3M@4vsnZ?LEl=R4lu_?VI)M9MfnD@d^>Za1X!AzD;q0Hc7li$2FoB5 zUgu7YbJ7GBI~!8ZW)av%vq|o)fTiAngov^z-b5BZ*L3`Hx$Mq%u^dZK8iPgNE(qu4 zgSIu=dBg!y&t?(KOUSl$V}a%LNfIK;(&`qn6nd%PwZPzBd7axy=EWz2!E)f+PGmW4 zzZahliPa(XQnJ{4&A~4(eH=|fL|K~WB8%beAzNXM>HO2*g=49{#9;BS-GVG7_u7wy zHTEWg)U#Ox=f38PQ9&>--tiE9%Bf62z@2fjPbVBM-+a;`rSd zFsF&mB&3Pvq;GzD30yy0yMTlV&1u~U-c_&4G#@`$m@>;rKm+E~axd>en|0|0kHMU7 zuOUg&nv>uFUHh->2j+DBFB03hIfZqIMsq6o=g>4TrwM)}P%3ju9&U){v_vropT2(| zNHRscM9OXKc)BCZ%fXz?LP<>1%*i(4-FWDpQIoDza~-GT3Zr|v-{sv;Q>kE425Y^O z4XI~$Pr-9ff$i@oLHCq(AR(gT@|ia-K?Zw90L%3`-qYJlcD3g@43;`eCHytE=yB`I z!1B)`QqN`)>;yia2TlMM>s2H~l;y)6oMS8NB76((rPTJ9IF@_27%bnbc=v2Y?Fs%0 zEYbO-UP_jz3tRDff-OIi5K)%myT}stYK08Ui{o5R?GBQ8$-c{AIb0lqt+3+5=&`_( zw3yUO$>Q5b4cA!WY7!#K^7XQf4<-U};lE>e(!U z^JU2PH%EYFcr^(TWy#M&mS^LNoPp()fBje`$r?NOjKQ+GA^=(9ZnwP!EP5fNp3Ned z7o#z{XMyGX9ugwTa_0fESbS6tg3Z?D_L{96OTdc;El?(sXMxjvBOZ|?5et+Z z%e#JU_o4S&u)y$FB&9J6Ts)iiFY594)1$xw%Zf={Ll*e;Sa0mu6>0C3zyc?KBvC(Q zfgbKY-N6DwYDuKH1$y=6S>TKT%~W7TWj0&;V-~p;C9pu>xO`lehvVnFf(2T(CLv9< zK%=ADlfVL_+mkS%1sWysUM~}Kzm+R2Or0(ypaBa^D&;=_72P8TEHJ%0Ns`tA1qW!m zyZ>rC3xZ=Z~OVI)ckwB>|aEi4tT43v4Eh3;xSPvzcVqL;>d_Hyw zrQJ`9!2-jLNKDf#aC3=G0CdmOCck8H9VhG+qkH;S{|A?8{ou+&u-4-(NWGNZvn1PM zKa9&68xkTmE;847uh_Yl9AWN^f+i}2>v!`cEWy!o$l`&Wj#q%t%{OX5wFfXT0k$N_Zz%~l@ zJ0AcRiz_5VlqK~MvJ6?>5zm)t`AJhambaA*mfe0l3!G`FI|5h+y(aZ+7QwuD_0(+x zEH6q)h$u_qV`S0u%l#eZ<+#IxQyk0V&kPpzse^D2mJ3iU0+vq;NIjcHaE*4_tmT`4 zW%~*eBFb{?39=LnZrvR=D68_LLpn*;*u$Cz%}KM)9M7E7E8iM`InAjdNh0Qy{JkDi zA|H%w4d!%5Cho`aG-giD`n)^$+s!l$0CP%_BXJFx)20i&`_CqR@^u1p`qYj@{ggSi zn(@jO%*m!Ri4->{XKS80_3FES0<5SD-ANF)q6FsjZ6xo`{V@T0N5Gt1`jU_)n$w~s z7Y2hl%^gU>gyxiVhWDx*g;}3_!NQCgLIN5vC+!iu2cXWWJn04Iq%o2tNo!7m1FYEH z+7Qgi(S*b{ZceK%#-lll~&lUvK^o~}c~utVroc0B=WUC)8kOW8d~cpb)X zBihU-A!6eq6T=&qp7ZCQ0T#_tn*@&K@^=Qy_zK>;-B%Qs=>W^jFQi^dmMn|tH8969 zGBbW0gec4YXE?{I%8up0VtcE0_^*EE>1J;&Xxly+3W_SmOK`EEjh)M;6_$mG;2W%8=BvSp>H6q_&SC zuy~r05K)#LFOX%--e+j%b(bAVIhM%H3>Nq2-gv%TJd!#KSX$mE^=uY_1zL79!M79J zy&xf?EI}`kB_s0KCs^EO{O~BP=Rv0xKEHG{cNfNQZ7v8+{ z&o6$%Ww1b-B_yRW3-omyfCnYNDiHrgz2r^e8nVDqqj~S-*yM5U5m?~YP!jc17U&gn z{#UR-%|sF@Zh;@S_C`#r#38bZ?_X4{DLq2L0;d#_kS1E-{S`5e zV1XCwNSM$9cc$}RVH8yzh!*Itq|INL%T_rrcKrR{7qYB!TekH7uVm>N$a^qN?$Vuw zV1WVsNs_b{C^*2Y3L1~V0*@M#*v2i;$n7Lr;Ka(h4q$C&cH^NCpvc~eG7%YwkKG;uh zd{27;ET7y+y_76Vqq^S*mN)B3h$zeAx5(1f^JXTnOmLeV%duRKX0VLSTZAks`f{Cs zC37{YXR`<_@QmKkPr%aIi-d@>xV}S{jt9)~?Zh9|M)p;e%u7ZrgJtzj-t(J7H9T=% z0{fAAHjCig@BH0W3s_urNr)(mQ!%ofC@z`}=A@xszma3P8P}jWrF5T&GnC^n<~^8G zkpW2(F{cBDJU@|ZeO?93$z&W!Y0R8fui;%-x|R1uAFiJ{OeS#+nbRv31MG4=tkO1v zIW3+|qJGMpocD!if;l~(Mp0c3$sla325N@Syn#pAv>GYCpW;HQumT1 zY0XJ+fR~?DT>*2dizcy+n^Sp6GMdx)+j_ZRPW$3Xpj76x{_8X}C!Y>S`#_hlOd^@0 zU83K6>=KVQ74`;m%1I+JO*1Ez-GlMjx(UA*1aciGF`m&q6^H8Ld3f!}2K-{loaLmR z-8}_6^q*rKhr_tE_aGsn<1&jkE~;v;v|v|z{V2X|SIJHgbArL*A-@cJ=HDBu@ttAC zZ%I9yMX;;wh>@KFEQ+5A+=G6k&p{I5x z^=uZwu4cQlS0u1xsFM&;7Ryp(37FC9Gq6l)k(a}<1f5~9m_GABmcXvf@tc)3?MXeG zMX<)Q4GQrM3stHlM3lw63|ZpzWOo2dqOHDVH_00FNoTNp3*lYYH0t`Y9_D4O8>wfr z2<$w_EWjS-W%?f^M3lv(99a%;OvR^m3|n{H#j#XpFj$h!f5UUXIC1q_U}>RD>e(#S z{O@hdFze_8EY@lyM3iOJ2V`-!)A$DSqI2+j4aZV)nZXiS$Ghtw@A5n?V0n{7>e(!U zb3ali8}Bd9%OWA7EJG`hW$%iG9&1 | tee -a /logs/librispeech_conformer_pytorch_09-11-2026-14-43-03.log +W0911 14:43:05.953000 9 site-packages/torch/distributed/run.py:803] +W0911 14:43:05.953000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0911 14:43:05.953000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0911 14:43:05.953000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-11 14:43:07.669609: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-11 14:43:07.669602: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-11 14:43:07.669603: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-11 14:43:07.669652: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789137787.690242 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789137787.690244 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789137787.690244 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789137787.690242 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789137787.696988 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789137787.697001 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789137787.697005 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789137787.697239 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789137787.713973 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137787.713972 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137787.713975 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137787.713994 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137787.713973 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137787.713995 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137787.713996 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137787.713997 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137787.713998 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137787.713998 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137787.713999 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137787.713999 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137787.714000 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137787.714001 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137787.714002 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137787.714003 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789137793.221241 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789137793.297709 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789137793.390258 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789137793.656871 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W911 14:43:15.264366144 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0911 14:43:16.419548 140066309203136 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/librispeech_conformer_pytorch. +I0911 14:43:16.419548 139939598574784 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/librispeech_conformer_pytorch. +I0911 14:43:16.419546 139885719721152 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/librispeech_conformer_pytorch. +I0911 14:43:16.419578 140540003882176 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/librispeech_conformer_pytorch. +I0911 14:43:16.480034 140540003882176 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/librispeech_conformer_pytorch/trial_1. +I0911 14:43:16.728301 140540003882176 submission_runner.py:242] Initializing dataset. +I0911 14:43:16.728503 140540003882176 input_pipeline.py:19] Loading split = train-clean-100 +I0911 14:43:16.775205 140540003882176 input_pipeline.py:19] Loading split = train-clean-360 +I0911 14:43:17.021064 140540003882176 input_pipeline.py:19] Loading split = train-other-500 +I0911 14:43:17.516072 140540003882176 submission_runner.py:251] Initializing model. +W0911 14:43:18.809744 139939598574784 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0911 14:43:18.809983 139885719721152 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0911 14:43:18.810470 140066309203136 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +I0911 14:43:18.811690 140066309203136 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0911 14:43:18.811696 139885719721152 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0911 14:43:18.811705 139939598574784 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0911 14:43:18.811800 140066309203136 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0911 14:43:18.811808 139885719721152 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0911 14:43:18.811818 139939598574784 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +W0911 14:43:18.812120 140540003882176 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +I0911 14:43:18.812289 140540003882176 submission_runner.py:294] Initializing optimizer. +I0911 14:43:18.812940 140540003882176 submission_runner.py:299] Initializing metrics bundle. +I0911 14:43:18.813073 140540003882176 submission_runner.py:321] Initializing checkpoint and logger. +I0911 14:43:18.813454 140540003882176 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_1/librispeech_conformer_pytorch/trial_1/meta_data_0.json. +I0911 14:43:18.813645 140540003882176 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0911 14:43:18.813704 140540003882176 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0911 14:43:19.087254 140540003882176 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_1/librispeech_conformer_pytorch/trial_1/flags_0.json. +I0911 14:43:19.102441 140540003882176 submission_runner.py:359] Starting training loop. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 14:43:30.022946 140521884321536 logging_writer.py:48] [0] global_step=0, grad_norm=0.5, loss=33.8528 +I0911 14:43:30.066067 140540003882176 submission.py:307] 0) loss = 33.853, grad_norm = 0.500 +I0911 14:43:30.874134 140540003882176 spec.py:333] Evaluating on the training split. +I0911 14:43:30.875125 140540003882176 input_pipeline.py:19] Loading split = train-clean-100 +I0911 14:43:30.899729 140540003882176 input_pipeline.py:19] Loading split = train-clean-360 +I0911 14:43:30.988670 140540003882176 input_pipeline.py:19] Loading split = train-other-500 +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 14:44:07.279702 140540003882176 spec.py:346] Evaluating on the validation split. +I0911 14:44:07.280881 140540003882176 input_pipeline.py:19] Loading split = dev-clean +I0911 14:44:07.299219 140540003882176 input_pipeline.py:19] Loading split = dev-other +I0911 14:44:26.913960 140540003882176 spec.py:363] Evaluating on the test split. +I0911 14:44:26.915099 140540003882176 input_pipeline.py:19] Loading split = test-clean +I0911 14:44:37.207069 140540003882176 submission_runner.py:516] Time since start: 78.10s, Step: 1, {'train/ctc_loss': 32.56430697422715, 'train/wer': 1.068215187960184, 'validation/ctc_loss': 31.449624522804903, 'validation/wer': 1.2736831941292908, 'validation/num_examples': 5348, 'test/ctc_loss': 31.58821859474805, 'test/wer': 1.2729876302480043, 'test/num_examples': 2472, 'score': 10.965123414993286, 'total_duration': 78.10448575019836, 'accumulated_submission_time': 10.965123414993286, 'accumulated_eval_time': 66.33343935012817, 'accumulated_logging_time': 0} +I0911 14:44:37.232124 140516691797760 logging_writer.py:48] [1] accumulated_eval_time=66.3334, accumulated_logging_time=0, accumulated_submission_time=10.9651, global_step=1, preemption_count=0, score=10.9651, test/ctc_loss=31.5882, test/num_examples=2472, test/wer=1.27299, total_duration=78.1045, train/ctc_loss=32.5643, train/wer=1.06822, validation/ctc_loss=31.4496, validation/num_examples=5348, validation/wer=1.27368 +I0911 14:44:39.452311 140516683405056 logging_writer.py:48] [1] global_step=1, grad_norm=0.5, loss=33.2051 +I0911 14:44:39.455451 140540003882176 submission.py:307] 1) loss = 33.205, grad_norm = 0.500 +I0911 14:44:40.097607 140516691797760 logging_writer.py:48] [2] global_step=2, grad_norm=0.5, loss=33.7304 +I0911 14:44:40.100639 140540003882176 submission.py:307] 2) loss = 33.730, grad_norm = 0.500 +I0911 14:44:40.542782 140516683405056 logging_writer.py:48] [3] global_step=3, grad_norm=0.5, loss=33.8648 +I0911 14:44:40.545655 140540003882176 submission.py:307] 3) loss = 33.865, grad_norm = 0.500 +I0911 14:44:40.988027 140516691797760 logging_writer.py:48] [4] global_step=4, grad_norm=0.5, loss=33.2787 +I0911 14:44:40.990854 140540003882176 submission.py:307] 4) loss = 33.279, grad_norm = 0.500 +I0911 14:44:41.433395 140516683405056 logging_writer.py:48] [5] global_step=5, grad_norm=0.5, loss=33.5761 +I0911 14:44:41.436182 140540003882176 submission.py:307] 5) loss = 33.576, grad_norm = 0.500 +I0911 14:44:41.878760 140516691797760 logging_writer.py:48] [6] global_step=6, grad_norm=0.5, loss=33.6898 +I0911 14:44:41.881637 140540003882176 submission.py:307] 6) loss = 33.690, grad_norm = 0.500 +I0911 14:44:42.324148 140516683405056 logging_writer.py:48] [7] global_step=7, grad_norm=0.5, loss=32.6531 +I0911 14:44:42.327098 140540003882176 submission.py:307] 7) loss = 32.653, grad_norm = 0.500 +I0911 14:44:42.769805 140516691797760 logging_writer.py:48] [8] global_step=8, grad_norm=0.5, loss=33.0357 +I0911 14:44:42.772622 140540003882176 submission.py:307] 8) loss = 33.036, grad_norm = 0.500 +I0911 14:44:43.398108 140516683405056 logging_writer.py:48] [9] global_step=9, grad_norm=0.5, loss=32.6784 +I0911 14:44:43.401068 140540003882176 submission.py:307] 9) loss = 32.678, grad_norm = 0.500 +I0911 14:44:45.150708 140516691797760 logging_writer.py:48] [10] global_step=10, grad_norm=0.5, loss=32.6442 +I0911 14:44:45.153657 140540003882176 submission.py:307] 10) loss = 32.644, grad_norm = 0.500 +I0911 14:44:46.163589 140516683405056 logging_writer.py:48] [11] global_step=11, grad_norm=0.5, loss=32.7882 +I0911 14:44:46.166472 140540003882176 submission.py:307] 11) loss = 32.788, grad_norm = 0.500 +I0911 14:44:46.608059 140516691797760 logging_writer.py:48] [12] global_step=12, grad_norm=0.5, loss=33.0092 +I0911 14:44:46.611155 140540003882176 submission.py:307] 12) loss = 33.009, grad_norm = 0.500 +I0911 14:44:49.320827 140516683405056 logging_writer.py:48] [13] global_step=13, grad_norm=0.5, loss=32.5735 +I0911 14:44:49.323742 140540003882176 submission.py:307] 13) loss = 32.573, grad_norm = 0.500 +I0911 14:44:50.524338 140516691797760 logging_writer.py:48] [14] global_step=14, grad_norm=0.5, loss=32.4328 +I0911 14:44:50.527200 140540003882176 submission.py:307] 14) loss = 32.433, grad_norm = 0.500 +I0911 14:44:51.850344 140516683405056 logging_writer.py:48] [15] global_step=15, grad_norm=0.5, loss=31.5621 +I0911 14:44:51.853179 140540003882176 submission.py:307] 15) loss = 31.562, grad_norm = 0.500 +I0911 14:44:52.294881 140516691797760 logging_writer.py:48] [16] global_step=16, grad_norm=0.5, loss=31.9342 +I0911 14:44:52.297738 140540003882176 submission.py:307] 16) loss = 31.934, grad_norm = 0.500 +I0911 14:44:54.513264 140516683405056 logging_writer.py:48] [17] global_step=17, grad_norm=0.5, loss=31.3439 +I0911 14:44:54.516099 140540003882176 submission.py:307] 17) loss = 31.344, grad_norm = 0.500 +I0911 14:44:56.289686 140516691797760 logging_writer.py:48] [18] global_step=18, grad_norm=0.5, loss=30.9348 +I0911 14:44:56.292630 140540003882176 submission.py:307] 18) loss = 30.935, grad_norm = 0.500 +I0911 14:44:57.423506 140516683405056 logging_writer.py:48] [19] global_step=19, grad_norm=0.5, loss=29.9155 +I0911 14:44:57.426354 140540003882176 submission.py:307] 19) loss = 29.916, grad_norm = 0.500 +I0911 14:44:57.928551 140516691797760 logging_writer.py:48] [20] global_step=20, grad_norm=0.5, loss=28.527 +I0911 14:44:57.931414 140540003882176 submission.py:307] 20) loss = 28.527, grad_norm = 0.500 +I0911 14:44:59.854667 140516683405056 logging_writer.py:48] [21] global_step=21, grad_norm=0.5, loss=27.4096 +I0911 14:44:59.857434 140540003882176 submission.py:307] 21) loss = 27.410, grad_norm = 0.500 +I0911 14:45:02.134581 140516691797760 logging_writer.py:48] [22] global_step=22, grad_norm=0.5, loss=26.4012 +I0911 14:45:02.137591 140540003882176 submission.py:307] 22) loss = 26.401, grad_norm = 0.500 +I0911 14:45:02.725574 140516683405056 logging_writer.py:48] [23] global_step=23, grad_norm=0.5, loss=24.0774 +I0911 14:45:02.728465 140540003882176 submission.py:307] 23) loss = 24.077, grad_norm = 0.500 +I0911 14:45:03.255881 140516691797760 logging_writer.py:48] [24] global_step=24, grad_norm=0.5, loss=22.4814 +I0911 14:45:03.258691 140540003882176 submission.py:307] 24) loss = 22.481, grad_norm = 0.500 +I0911 14:45:05.573398 140516683405056 logging_writer.py:48] [25] global_step=25, grad_norm=0.5, loss=20.5337 +I0911 14:45:05.576386 140540003882176 submission.py:307] 25) loss = 20.534, grad_norm = 0.500 +I0911 14:45:08.071429 140516691797760 logging_writer.py:48] [26] global_step=26, grad_norm=0.5, loss=18.5169 +I0911 14:45:08.074317 140540003882176 submission.py:307] 26) loss = 18.517, grad_norm = 0.500 +I0911 14:45:08.516830 140516683405056 logging_writer.py:48] [27] global_step=27, grad_norm=0.5, loss=16.5803 +I0911 14:45:08.519644 140540003882176 submission.py:307] 27) loss = 16.580, grad_norm = 0.500 +I0911 14:45:08.961910 140516691797760 logging_writer.py:48] [28] global_step=28, grad_norm=0.5, loss=14.5444 +I0911 14:45:08.964812 140540003882176 submission.py:307] 28) loss = 14.544, grad_norm = 0.500 +I0911 14:45:11.286096 140516683405056 logging_writer.py:48] [29] global_step=29, grad_norm=0.5, loss=12.656 +I0911 14:45:11.288875 140540003882176 submission.py:307] 29) loss = 12.656, grad_norm = 0.500 +I0911 14:45:13.516465 140516691797760 logging_writer.py:48] [30] global_step=30, grad_norm=0.5, loss=11.0136 +I0911 14:45:13.519319 140540003882176 submission.py:307] 30) loss = 11.014, grad_norm = 0.500 +I0911 14:45:13.961588 140516683405056 logging_writer.py:48] [31] global_step=31, grad_norm=0.5, loss=9.64463 +I0911 14:45:13.964413 140540003882176 submission.py:307] 31) loss = 9.645, grad_norm = 0.500 +I0911 14:45:14.406533 140516691797760 logging_writer.py:48] [32] global_step=32, grad_norm=0.5, loss=8.5662 +I0911 14:45:14.409355 140540003882176 submission.py:307] 32) loss = 8.566, grad_norm = 0.500 +I0911 14:45:16.767007 140516683405056 logging_writer.py:48] [33] global_step=33, grad_norm=0.5, loss=7.82868 +I0911 14:45:16.769891 140540003882176 submission.py:307] 33) loss = 7.829, grad_norm = 0.500 +I0911 14:45:19.074110 140516691797760 logging_writer.py:48] [34] global_step=34, grad_norm=0.5, loss=7.37142 +I0911 14:45:19.077074 140540003882176 submission.py:307] 34) loss = 7.371, grad_norm = 0.500 +I0911 14:45:19.520035 140516683405056 logging_writer.py:48] [35] global_step=35, grad_norm=0.5, loss=7.18609 +I0911 14:45:19.522866 140540003882176 submission.py:307] 35) loss = 7.186, grad_norm = 0.500 +I0911 14:45:19.964919 140516691797760 logging_writer.py:48] [36] global_step=36, grad_norm=0.5, loss=7.18443 +I0911 14:45:19.967854 140540003882176 submission.py:307] 36) loss = 7.184, grad_norm = 0.500 +I0911 14:45:21.710716 140516683405056 logging_writer.py:48] [37] global_step=37, grad_norm=0.5, loss=7.25404 +I0911 14:45:21.713713 140540003882176 submission.py:307] 37) loss = 7.254, grad_norm = 0.500 +I0911 14:45:24.439288 140516691797760 logging_writer.py:48] [38] global_step=38, grad_norm=0.5, loss=7.28419 +I0911 14:45:24.447397 140540003882176 submission.py:307] 38) loss = 7.284, grad_norm = 0.500 +I0911 14:45:24.890188 140516683405056 logging_writer.py:48] [39] global_step=39, grad_norm=0.5, loss=7.26474 +I0911 14:45:24.893086 140540003882176 submission.py:307] 39) loss = 7.265, grad_norm = 0.500 +I0911 14:45:25.335789 140516691797760 logging_writer.py:48] [40] global_step=40, grad_norm=0.5, loss=7.19829 +I0911 14:45:25.338618 140540003882176 submission.py:307] 40) loss = 7.198, grad_norm = 0.500 +I0911 14:45:26.956492 140516683405056 logging_writer.py:48] [41] global_step=41, grad_norm=0.5, loss=7.10598 +I0911 14:45:26.959319 140540003882176 submission.py:307] 41) loss = 7.106, grad_norm = 0.500 +I0911 14:45:29.958542 140516691797760 logging_writer.py:48] [42] global_step=42, grad_norm=0.5, loss=7.02456 +I0911 14:45:29.961484 140540003882176 submission.py:307] 42) loss = 7.025, grad_norm = 0.500 +I0911 14:45:30.415607 140516683405056 logging_writer.py:48] [43] global_step=43, grad_norm=0.5, loss=6.98174 +I0911 14:45:30.418611 140540003882176 submission.py:307] 43) loss = 6.982, grad_norm = 0.500 +I0911 14:45:30.861202 140516691797760 logging_writer.py:48] [44] global_step=44, grad_norm=0.5, loss=6.97104 +I0911 14:45:30.864123 140540003882176 submission.py:307] 44) loss = 6.971, grad_norm = 0.500 +I0911 14:45:32.518859 140516683405056 logging_writer.py:48] [45] global_step=45, grad_norm=0.5, loss=6.95665 +I0911 14:45:32.521776 140540003882176 submission.py:307] 45) loss = 6.957, grad_norm = 0.500 +I0911 14:45:35.598009 140516691797760 logging_writer.py:48] [46] global_step=46, grad_norm=0.5, loss=6.9048 +I0911 14:45:35.600893 140540003882176 submission.py:307] 46) loss = 6.905, grad_norm = 0.500 +I0911 14:45:36.043850 140516683405056 logging_writer.py:48] [47] global_step=47, grad_norm=0.5, loss=6.87298 +I0911 14:45:36.046721 140540003882176 submission.py:307] 47) loss = 6.873, grad_norm = 0.500 +I0911 14:45:36.490080 140516691797760 logging_writer.py:48] [48] global_step=48, grad_norm=0.5, loss=6.81049 +I0911 14:45:36.492909 140540003882176 submission.py:307] 48) loss = 6.810, grad_norm = 0.500 +I0911 14:45:38.265063 140516683405056 logging_writer.py:48] [49] global_step=49, grad_norm=0.5, loss=6.78761 +I0911 14:45:38.267902 140540003882176 submission.py:307] 49) loss = 6.788, grad_norm = 0.500 +I0911 14:45:40.958902 140516691797760 logging_writer.py:48] [50] global_step=50, grad_norm=0.5, loss=6.75703 +I0911 14:45:40.962050 140540003882176 submission.py:307] 50) loss = 6.757, grad_norm = 0.500 +I0911 14:45:41.404890 140516683405056 logging_writer.py:48] [51] global_step=51, grad_norm=0.5, loss=6.6878 +I0911 14:45:41.407818 140540003882176 submission.py:307] 51) loss = 6.688, grad_norm = 0.500 +I0911 14:45:41.849946 140516691797760 logging_writer.py:48] [52] global_step=52, grad_norm=0.5, loss=6.66775 +I0911 14:45:41.852819 140540003882176 submission.py:307] 52) loss = 6.668, grad_norm = 0.500 +I0911 14:45:43.880330 140516683405056 logging_writer.py:48] [53] global_step=53, grad_norm=0.5, loss=6.65202 +I0911 14:45:43.883287 140540003882176 submission.py:307] 53) loss = 6.652, grad_norm = 0.500 +I0911 14:45:46.082314 140516691797760 logging_writer.py:48] [54] global_step=54, grad_norm=0.5, loss=6.61864 +I0911 14:45:46.085151 140540003882176 submission.py:307] 54) loss = 6.619, grad_norm = 0.500 +I0911 14:45:46.527270 140516683405056 logging_writer.py:48] [55] global_step=55, grad_norm=0.5, loss=6.578 +I0911 14:45:46.530190 140540003882176 submission.py:307] 55) loss = 6.578, grad_norm = 0.500 +I0911 14:45:47.173365 140516691797760 logging_writer.py:48] [56] global_step=56, grad_norm=0.5, loss=6.54822 +I0911 14:45:47.176227 140540003882176 submission.py:307] 56) loss = 6.548, grad_norm = 0.500 +I0911 14:45:49.453278 140516683405056 logging_writer.py:48] [57] global_step=57, grad_norm=0.5, loss=6.52059 +I0911 14:45:49.456213 140540003882176 submission.py:307] 57) loss = 6.521, grad_norm = 0.500 +I0911 14:45:51.442829 140516691797760 logging_writer.py:48] [58] global_step=58, grad_norm=0.5, loss=6.48128 +I0911 14:45:51.445826 140540003882176 submission.py:307] 58) loss = 6.481, grad_norm = 0.500 +I0911 14:45:51.888803 140516683405056 logging_writer.py:48] [59] global_step=59, grad_norm=0.5, loss=6.46436 +I0911 14:45:51.891653 140540003882176 submission.py:307] 59) loss = 6.464, grad_norm = 0.500 +I0911 14:45:52.654526 140516691797760 logging_writer.py:48] [60] global_step=60, grad_norm=0.5, loss=6.44828 +I0911 14:45:52.657330 140540003882176 submission.py:307] 60) loss = 6.448, grad_norm = 0.500 +I0911 14:45:55.264158 140516683405056 logging_writer.py:48] [61] global_step=61, grad_norm=0.5, loss=6.41875 +I0911 14:45:55.267005 140540003882176 submission.py:307] 61) loss = 6.419, grad_norm = 0.500 +I0911 14:45:56.798179 140516691797760 logging_writer.py:48] [62] global_step=62, grad_norm=0.5, loss=6.39103 +I0911 14:45:56.801114 140540003882176 submission.py:307] 62) loss = 6.391, grad_norm = 0.500 +I0911 14:45:57.369265 140516683405056 logging_writer.py:48] [63] global_step=63, grad_norm=0.5, loss=6.35958 +I0911 14:45:57.372071 140540003882176 submission.py:307] 63) loss = 6.360, grad_norm = 0.500 +I0911 14:45:58.162160 140516691797760 logging_writer.py:48] [64] global_step=64, grad_norm=0.5, loss=6.33572 +I0911 14:45:58.165148 140540003882176 submission.py:307] 64) loss = 6.336, grad_norm = 0.500 +I0911 14:46:00.865971 140516683405056 logging_writer.py:48] [65] global_step=65, grad_norm=0.5, loss=6.33765 +I0911 14:46:00.868820 140540003882176 submission.py:307] 65) loss = 6.338, grad_norm = 0.500 +I0911 14:46:02.554046 140516691797760 logging_writer.py:48] [66] global_step=66, grad_norm=0.5, loss=6.30197 +I0911 14:46:02.556909 140540003882176 submission.py:307] 66) loss = 6.302, grad_norm = 0.500 +I0911 14:46:02.999546 140516683405056 logging_writer.py:48] [67] global_step=67, grad_norm=0.5, loss=6.27609 +I0911 14:46:03.002450 140540003882176 submission.py:307] 67) loss = 6.276, grad_norm = 0.500 +I0911 14:46:03.865470 140516691797760 logging_writer.py:48] [68] global_step=68, grad_norm=0.5, loss=6.28132 +I0911 14:46:03.868366 140540003882176 submission.py:307] 68) loss = 6.281, grad_norm = 0.500 +I0911 14:46:05.977698 140516683405056 logging_writer.py:48] [69] global_step=69, grad_norm=0.5, loss=6.27631 +I0911 14:46:05.980573 140540003882176 submission.py:307] 69) loss = 6.276, grad_norm = 0.500 +I0911 14:46:07.869754 140516691797760 logging_writer.py:48] [70] global_step=70, grad_norm=0.5, loss=6.23777 +I0911 14:46:07.872571 140540003882176 submission.py:307] 70) loss = 6.238, grad_norm = 0.500 +I0911 14:46:08.315573 140516683405056 logging_writer.py:48] [71] global_step=71, grad_norm=0.5, loss=6.23944 +I0911 14:46:08.318447 140540003882176 submission.py:307] 71) loss = 6.239, grad_norm = 0.500 +I0911 14:46:09.725435 140516691797760 logging_writer.py:48] [72] global_step=72, grad_norm=0.5, loss=6.20528 +I0911 14:46:09.728291 140540003882176 submission.py:307] 72) loss = 6.205, grad_norm = 0.500 +I0911 14:46:11.469727 140516683405056 logging_writer.py:48] [73] global_step=73, grad_norm=0.5, loss=6.20009 +I0911 14:46:11.472634 140540003882176 submission.py:307] 73) loss = 6.200, grad_norm = 0.500 +I0911 14:46:13.311546 140516691797760 logging_writer.py:48] [74] global_step=74, grad_norm=0.5, loss=6.2027 +I0911 14:46:13.314416 140540003882176 submission.py:307] 74) loss = 6.203, grad_norm = 0.500 +I0911 14:46:13.757054 140516683405056 logging_writer.py:48] [75] global_step=75, grad_norm=0.5, loss=6.18128 +I0911 14:46:13.759939 140540003882176 submission.py:307] 75) loss = 6.181, grad_norm = 0.500 +I0911 14:46:15.442296 140516691797760 logging_writer.py:48] [76] global_step=76, grad_norm=0.5, loss=6.17227 +I0911 14:46:15.445213 140540003882176 submission.py:307] 76) loss = 6.172, grad_norm = 0.500 +I0911 14:46:16.742163 140516683405056 logging_writer.py:48] [77] global_step=77, grad_norm=0.5, loss=6.15774 +I0911 14:46:16.744988 140540003882176 submission.py:307] 77) loss = 6.158, grad_norm = 0.500 +I0911 14:46:19.418347 140516691797760 logging_writer.py:48] [78] global_step=78, grad_norm=0.5, loss=6.15941 +I0911 14:46:19.421213 140540003882176 submission.py:307] 78) loss = 6.159, grad_norm = 0.500 +I0911 14:46:19.864478 140516683405056 logging_writer.py:48] [79] global_step=79, grad_norm=0.5, loss=6.15644 +I0911 14:46:19.867412 140540003882176 submission.py:307] 79) loss = 6.156, grad_norm = 0.500 +I0911 14:46:20.843907 140516691797760 logging_writer.py:48] [80] global_step=80, grad_norm=0.5, loss=6.13468 +I0911 14:46:20.846813 140540003882176 submission.py:307] 80) loss = 6.135, grad_norm = 0.500 +I0911 14:46:22.040347 140516683405056 logging_writer.py:48] [81] global_step=81, grad_norm=0.5, loss=6.11274 +I0911 14:46:22.043166 140540003882176 submission.py:307] 81) loss = 6.113, grad_norm = 0.500 +I0911 14:46:24.575010 140516691797760 logging_writer.py:48] [82] global_step=82, grad_norm=0.499999, loss=6.10201 +I0911 14:46:24.578021 140540003882176 submission.py:307] 82) loss = 6.102, grad_norm = 0.500 +I0911 14:46:25.021364 140516683405056 logging_writer.py:48] [83] global_step=83, grad_norm=0.5, loss=6.11788 +I0911 14:46:25.024229 140540003882176 submission.py:307] 83) loss = 6.118, grad_norm = 0.500 +I0911 14:46:26.107533 140516691797760 logging_writer.py:48] [84] global_step=84, grad_norm=0.5, loss=6.10421 +I0911 14:46:26.110386 140540003882176 submission.py:307] 84) loss = 6.104, grad_norm = 0.500 +I0911 14:46:27.406777 140516683405056 logging_writer.py:48] [85] global_step=85, grad_norm=0.5, loss=6.06445 +I0911 14:46:27.409586 140540003882176 submission.py:307] 85) loss = 6.064, grad_norm = 0.500 +I0911 14:46:29.989861 140516691797760 logging_writer.py:48] [86] global_step=86, grad_norm=0.5, loss=6.07449 +I0911 14:46:29.992697 140540003882176 submission.py:307] 86) loss = 6.074, grad_norm = 0.500 +I0911 14:46:30.435659 140516683405056 logging_writer.py:48] [87] global_step=87, grad_norm=0.5, loss=6.05189 +I0911 14:46:30.438713 140540003882176 submission.py:307] 87) loss = 6.052, grad_norm = 0.500 +I0911 14:46:31.409496 140516691797760 logging_writer.py:48] [88] global_step=88, grad_norm=0.5, loss=6.06839 +I0911 14:46:31.412393 140540003882176 submission.py:307] 88) loss = 6.068, grad_norm = 0.500 +I0911 14:46:32.596609 140516683405056 logging_writer.py:48] [89] global_step=89, grad_norm=0.5, loss=6.06728 +I0911 14:46:32.599700 140540003882176 submission.py:307] 89) loss = 6.067, grad_norm = 0.500 +I0911 14:46:36.072573 140516691797760 logging_writer.py:48] [90] global_step=90, grad_norm=0.5, loss=6.03502 +I0911 14:46:36.075451 140540003882176 submission.py:307] 90) loss = 6.035, grad_norm = 0.500 +I0911 14:46:36.519244 140516683405056 logging_writer.py:48] [91] global_step=91, grad_norm=0.5, loss=6.04821 +I0911 14:46:36.522216 140540003882176 submission.py:307] 91) loss = 6.048, grad_norm = 0.500 +I0911 14:46:37.171119 140516691797760 logging_writer.py:48] [92] global_step=92, grad_norm=0.5, loss=6.03251 +I0911 14:46:37.174057 140540003882176 submission.py:307] 92) loss = 6.033, grad_norm = 0.500 +I0911 14:46:37.777921 140516683405056 logging_writer.py:48] [93] global_step=93, grad_norm=0.5, loss=6.01521 +I0911 14:46:37.780841 140540003882176 submission.py:307] 93) loss = 6.015, grad_norm = 0.500 +I0911 14:46:41.545966 140516691797760 logging_writer.py:48] [94] global_step=94, grad_norm=0.5, loss=6.00538 +I0911 14:46:41.548778 140540003882176 submission.py:307] 94) loss = 6.005, grad_norm = 0.500 +I0911 14:46:41.992660 140516683405056 logging_writer.py:48] [95] global_step=95, grad_norm=0.5, loss=6.02245 +I0911 14:46:41.995749 140540003882176 submission.py:307] 95) loss = 6.022, grad_norm = 0.500 +I0911 14:46:42.758183 140516691797760 logging_writer.py:48] [96] global_step=96, grad_norm=0.5, loss=6.01979 +I0911 14:46:42.761010 140540003882176 submission.py:307] 96) loss = 6.020, grad_norm = 0.500 +I0911 14:46:43.203841 140516683405056 logging_writer.py:48] [97] global_step=97, grad_norm=0.5, loss=5.98432 +I0911 14:46:43.206703 140540003882176 submission.py:307] 97) loss = 5.984, grad_norm = 0.500 +I0911 14:46:47.196048 140516691797760 logging_writer.py:48] [98] global_step=98, grad_norm=0.5, loss=5.98571 +I0911 14:46:47.198909 140540003882176 submission.py:307] 98) loss = 5.986, grad_norm = 0.500 +I0911 14:46:47.643212 140516683405056 logging_writer.py:48] [99] global_step=99, grad_norm=0.5, loss=5.99313 +I0911 14:46:47.646195 140540003882176 submission.py:307] 99) loss = 5.993, grad_norm = 0.500 +I0911 14:46:48.089247 140516691797760 logging_writer.py:48] [100] global_step=100, grad_norm=0.5, loss=5.99319 +I0911 14:46:48.092200 140540003882176 submission.py:307] 100) loss = 5.993, grad_norm = 0.500 +I0911 14:55:56.251420 140516683405056 logging_writer.py:48] [500] global_step=500, grad_norm=0.5, loss=5.73399 +I0911 14:55:56.254718 140540003882176 submission.py:307] 500) loss = 5.734, grad_norm = 0.500 +I0911 15:07:32.166811 140516691797760 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.499999, loss=3.46954 +I0911 15:07:32.170408 140540003882176 submission.py:307] 1000) loss = 3.470, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 15:12:16.214250 140516691797760 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.499999, loss=2.75357 +I0911 15:12:16.221578 140540003882176 submission.py:307] 1500) loss = 2.754, grad_norm = 0.500 +I0911 15:13:45.491310 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 15:13:59.769816 140540003882176 spec.py:346] Evaluating on the validation split. +I0911 15:14:19.671837 140540003882176 spec.py:363] Evaluating on the test split. +I0911 15:14:29.989682 140540003882176 submission_runner.py:516] Time since start: 1870.89s, Step: 1616, {'train/ctc_loss': 4.018316224717581, 'train/wer': 0.765940448686114, 'validation/ctc_loss': 4.212096738748242, 'validation/wer': 0.7391976053686091, 'validation/num_examples': 5348, 'test/ctc_loss': 3.992824133815085, 'test/wer': 0.7336542562915118, 'test/num_examples': 2472, 'score': 1757.0212490558624, 'total_duration': 1870.887146472931, 'accumulated_submission_time': 1757.0212490558624, 'accumulated_eval_time': 110.83208703994751, 'accumulated_logging_time': 0.03430986404418945} +I0911 15:14:30.019547 140516691797760 logging_writer.py:48] [1616] accumulated_eval_time=110.832, accumulated_logging_time=0.0343099, accumulated_submission_time=1757.02, global_step=1616, preemption_count=0, score=1757.02, test/ctc_loss=3.99282, test/num_examples=2472, test/wer=0.733654, total_duration=1870.89, train/ctc_loss=4.01832, train/wer=0.76594, validation/ctc_loss=4.2121, validation/num_examples=5348, validation/wer=0.739198 +I0911 15:22:40.504012 140516683405056 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.499999, loss=2.30621 +I0911 15:22:40.507756 140540003882176 submission.py:307] 2000) loss = 2.306, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 15:27:26.520194 140516691797760 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.499999, loss=2.2095 +I0911 15:27:26.526450 140540003882176 submission.py:307] 2500) loss = 2.210, grad_norm = 0.500 +I0911 15:34:40.534228 140516683405056 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.499999, loss=1.98994 +I0911 15:34:40.537818 140540003882176 submission.py:307] 3000) loss = 1.990, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 15:39:29.939304 140516691797760 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.499999, loss=1.85125 +I0911 15:39:29.946027 140540003882176 submission.py:307] 3500) loss = 1.851, grad_norm = 0.500 +I0911 15:43:39.148920 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 15:43:55.014656 140540003882176 spec.py:346] Evaluating on the validation split. +I0911 15:44:14.783841 140540003882176 spec.py:363] Evaluating on the test split. +I0911 15:44:24.990725 140540003882176 submission_runner.py:516] Time since start: 3665.89s, Step: 3833, {'train/ctc_loss': 0.7888560762362172, 'train/wer': 0.2638028199425647, 'validation/ctc_loss': 0.9988812796677216, 'validation/wer': 0.30181045720079175, 'validation/num_examples': 5348, 'test/ctc_loss': 0.7281453642170462, 'test/wer': 0.2421952755265777, 'test/num_examples': 2472, 'score': 3503.4649250507355, 'total_duration': 3665.88813328743, 'accumulated_submission_time': 3503.4649250507355, 'accumulated_eval_time': 156.67479920387268, 'accumulated_logging_time': 0.0733029842376709} +I0911 15:44:25.037490 140516691797760 logging_writer.py:48] [3833] accumulated_eval_time=156.675, accumulated_logging_time=0.073303, accumulated_submission_time=3503.46, global_step=3833, preemption_count=0, score=3503.46, test/ctc_loss=0.728145, test/num_examples=2472, test/wer=0.242195, total_duration=3665.89, train/ctc_loss=0.788856, train/wer=0.263803, validation/ctc_loss=0.998881, validation/num_examples=5348, validation/wer=0.30181 +I0911 15:47:23.632865 140516683405056 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.499999, loss=1.89868 +I0911 15:47:23.636221 140540003882176 submission.py:307] 4000) loss = 1.899, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 15:52:34.117488 140516691797760 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.499999, loss=1.81688 +I0911 15:52:34.130413 140540003882176 submission.py:307] 4500) loss = 1.817, grad_norm = 0.500 +I0911 15:59:26.191369 140516683405056 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.499999, loss=1.68535 +I0911 15:59:26.195237 140540003882176 submission.py:307] 5000) loss = 1.685, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 16:04:47.399941 140516691797760 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.499999, loss=1.72244 +I0911 16:04:47.407977 140540003882176 submission.py:307] 5500) loss = 1.722, grad_norm = 0.500 +I0911 16:11:16.306460 140516683405056 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.499999, loss=1.62225 +I0911 16:11:16.310152 140540003882176 submission.py:307] 6000) loss = 1.622, grad_norm = 0.500 +I0911 16:13:34.413354 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 16:13:49.100694 140540003882176 spec.py:346] Evaluating on the validation split. +I0911 16:14:08.744936 140540003882176 spec.py:363] Evaluating on the test split. +I0911 16:14:18.959794 140540003882176 submission_runner.py:516] Time since start: 5459.86s, Step: 6123, {'train/ctc_loss': 0.4452975969018467, 'train/wer': 0.1541953663118347, 'validation/ctc_loss': 0.668249043255224, 'validation/wer': 0.2019601216627239, 'validation/num_examples': 5348, 'test/ctc_loss': 0.4199425920623911, 'test/wer': 0.14159202161152074, 'test/num_examples': 2472, 'score': 5250.175142049789, 'total_duration': 5459.857209920883, 'accumulated_submission_time': 5250.175142049789, 'accumulated_eval_time': 201.22112226486206, 'accumulated_logging_time': 0.12943315505981445} +I0911 16:14:19.062895 140516691797760 logging_writer.py:48] [6123] accumulated_eval_time=201.221, accumulated_logging_time=0.129433, accumulated_submission_time=5250.18, global_step=6123, preemption_count=0, score=5250.18, test/ctc_loss=0.419943, test/num_examples=2472, test/wer=0.141592, total_duration=5459.86, train/ctc_loss=0.445298, train/wer=0.154195, validation/ctc_loss=0.668249, validation/num_examples=5348, validation/wer=0.20196 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 16:17:41.798593 140516691797760 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.499999, loss=1.58638 +I0911 16:17:41.805332 140540003882176 submission.py:307] 6500) loss = 1.586, grad_norm = 0.500 +I0911 16:24:01.460798 140516683405056 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.499999, loss=1.58504 +I0911 16:24:01.464400 140540003882176 submission.py:307] 7000) loss = 1.585, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 16:29:59.672223 140516691797760 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.499999, loss=1.59838 +I0911 16:29:59.679201 140540003882176 submission.py:307] 7500) loss = 1.598, grad_norm = 0.500 +I0911 16:35:50.696768 140516683405056 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.499999, loss=1.52558 +I0911 16:35:50.700461 140540003882176 submission.py:307] 8000) loss = 1.526, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 16:42:14.127601 140516691797760 logging_writer.py:48] [8500] global_step=8500, grad_norm=0.499999, loss=1.45069 +I0911 16:42:14.134162 140540003882176 submission.py:307] 8500) loss = 1.451, grad_norm = 0.500 +I0911 16:43:27.248045 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 16:43:41.351571 140540003882176 spec.py:346] Evaluating on the validation split. +I0911 16:44:01.256180 140540003882176 spec.py:363] Evaluating on the test split. +I0911 16:44:11.500095 140540003882176 submission_runner.py:516] Time since start: 7252.40s, Step: 8655, {'train/ctc_loss': 0.3602500813857133, 'train/wer': 0.12670308552674195, 'validation/ctc_loss': 0.5876604156463482, 'validation/wer': 0.1749336165693043, 'validation/num_examples': 5348, 'test/ctc_loss': 0.3605698855571327, 'test/wer': 0.12067109459102635, 'test/num_examples': 2472, 'score': 6995.479728221893, 'total_duration': 7252.397528409958, 'accumulated_submission_time': 6995.479728221893, 'accumulated_eval_time': 245.47307872772217, 'accumulated_logging_time': 0.24185848236083984} +I0911 16:44:11.584688 140516691797760 logging_writer.py:48] [8655] accumulated_eval_time=245.473, accumulated_logging_time=0.241858, accumulated_submission_time=6995.48, global_step=8655, preemption_count=0, score=6995.48, test/ctc_loss=0.36057, test/num_examples=2472, test/wer=0.120671, total_duration=7252.4, train/ctc_loss=0.36025, train/wer=0.126703, validation/ctc_loss=0.58766, validation/num_examples=5348, validation/wer=0.174934 +I0911 16:48:40.993018 140516683405056 logging_writer.py:48] [9000] global_step=9000, grad_norm=0.499999, loss=1.49941 +I0911 16:48:40.996702 140540003882176 submission.py:307] 9000) loss = 1.499, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 16:55:17.914397 140516691797760 logging_writer.py:48] [9500] global_step=9500, grad_norm=0.499999, loss=1.55124 +I0911 16:55:17.921329 140540003882176 submission.py:307] 9500) loss = 1.551, grad_norm = 0.500 +I0911 17:00:33.500827 140516683405056 logging_writer.py:48] [10000] global_step=10000, grad_norm=0.499999, loss=1.44768 +I0911 17:00:33.504632 140540003882176 submission.py:307] 10000) loss = 1.448, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 17:07:26.340734 140516691797760 logging_writer.py:48] [10500] global_step=10500, grad_norm=0.499999, loss=1.44552 +I0911 17:07:26.347709 140540003882176 submission.py:307] 10500) loss = 1.446, grad_norm = 0.500 +I0911 17:12:25.450761 140516683405056 logging_writer.py:48] [11000] global_step=11000, grad_norm=0.499999, loss=1.47696 +I0911 17:12:25.454396 140540003882176 submission.py:307] 11000) loss = 1.477, grad_norm = 0.500 +I0911 17:13:20.634935 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 17:13:35.883543 140540003882176 spec.py:346] Evaluating on the validation split. +I0911 17:13:55.516148 140540003882176 spec.py:363] Evaluating on the test split. +I0911 17:14:05.811078 140540003882176 submission_runner.py:516] Time since start: 9046.71s, Step: 11059, {'train/ctc_loss': 0.31485508586721234, 'train/wer': 0.11292185779370803, 'validation/ctc_loss': 0.53931664759456, 'validation/wer': 0.16072997634335925, 'validation/num_examples': 5348, 'test/ctc_loss': 0.3231082700900058, 'test/wer': 0.10789511100278269, 'test/num_examples': 2472, 'score': 8741.733561992645, 'total_duration': 9046.708485126495, 'accumulated_submission_time': 8741.733561992645, 'accumulated_eval_time': 290.6490111351013, 'accumulated_logging_time': 0.3359074592590332} +I0911 17:14:05.921581 140516691797760 logging_writer.py:48] [11059] accumulated_eval_time=290.649, accumulated_logging_time=0.335907, accumulated_submission_time=8741.73, global_step=11059, preemption_count=0, score=8741.73, test/ctc_loss=0.323108, test/num_examples=2472, test/wer=0.107895, total_duration=9046.71, train/ctc_loss=0.314855, train/wer=0.112922, validation/ctc_loss=0.539317, validation/num_examples=5348, validation/wer=0.16073 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 17:20:15.908475 140516691797760 logging_writer.py:48] [11500] global_step=11500, grad_norm=0.499999, loss=1.4429 +I0911 17:20:15.915216 140540003882176 submission.py:307] 11500) loss = 1.443, grad_norm = 0.500 +I0911 17:25:08.340472 140516683405056 logging_writer.py:48] [12000] global_step=12000, grad_norm=0.499999, loss=1.35634 +I0911 17:25:08.344065 140540003882176 submission.py:307] 12000) loss = 1.356, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 17:32:29.539991 140516691797760 logging_writer.py:48] [12500] global_step=12500, grad_norm=0.499999, loss=1.44118 +I0911 17:32:29.546828 140540003882176 submission.py:307] 12500) loss = 1.441, grad_norm = 0.500 +I0911 17:37:04.176364 140516683405056 logging_writer.py:48] [13000] global_step=13000, grad_norm=0.499999, loss=1.47589 +I0911 17:37:04.180009 140540003882176 submission.py:307] 13000) loss = 1.476, grad_norm = 0.500 +I0911 17:43:16.302345 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 17:43:30.183891 140540003882176 spec.py:346] Evaluating on the validation split. +I0911 17:43:49.850051 140540003882176 spec.py:363] Evaluating on the test split. +I0911 17:44:00.215219 140540003882176 submission_runner.py:516] Time since start: 10841.11s, Step: 13355, {'train/ctc_loss': 0.28467586289995267, 'train/wer': 0.10354544080495758, 'validation/ctc_loss': 0.5182867067416366, 'validation/wer': 0.15474339786607444, 'validation/num_examples': 5348, 'test/ctc_loss': 0.3071018190689722, 'test/wer': 0.10257347713931712, 'test/num_examples': 2472, 'score': 10488.446098566055, 'total_duration': 10841.112599849701, 'accumulated_submission_time': 10488.446098566055, 'accumulated_eval_time': 334.5619010925293, 'accumulated_logging_time': 0.4564213752746582} +I0911 17:44:00.560699 140516691797760 logging_writer.py:48] [13355] accumulated_eval_time=334.562, accumulated_logging_time=0.456421, accumulated_submission_time=10488.4, global_step=13355, preemption_count=0, score=10488.4, test/ctc_loss=0.307102, test/num_examples=2472, test/wer=0.102573, total_duration=10841.1, train/ctc_loss=0.284676, train/wer=0.103545, validation/ctc_loss=0.518287, validation/num_examples=5348, validation/wer=0.154743 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 17:45:25.167405 140516691797760 logging_writer.py:48] [13500] global_step=13500, grad_norm=0.499999, loss=1.37811 +I0911 17:45:25.174327 140540003882176 submission.py:307] 13500) loss = 1.378, grad_norm = 0.500 +I0911 17:50:03.030983 140516683405056 logging_writer.py:48] [14000] global_step=14000, grad_norm=0.499999, loss=1.33047 +I0911 17:50:03.034620 140540003882176 submission.py:307] 14000) loss = 1.330, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 17:57:47.204151 140516691797760 logging_writer.py:48] [14500] global_step=14500, grad_norm=0.499999, loss=1.42513 +I0911 17:57:47.211843 140540003882176 submission.py:307] 14500) loss = 1.425, grad_norm = 0.500 +I0911 18:01:59.349476 140516683405056 logging_writer.py:48] [15000] global_step=15000, grad_norm=0.499999, loss=1.3908 +I0911 18:01:59.353251 140540003882176 submission.py:307] 15000) loss = 1.391, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 18:10:02.013175 140516691797760 logging_writer.py:48] [15500] global_step=15500, grad_norm=0.499999, loss=1.34453 +I0911 18:10:02.020111 140540003882176 submission.py:307] 15500) loss = 1.345, grad_norm = 0.500 +I0911 18:13:08.596125 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 18:13:23.485177 140540003882176 spec.py:346] Evaluating on the validation split. +I0911 18:13:43.372640 140540003882176 spec.py:363] Evaluating on the test split. +I0911 18:13:53.617445 140540003882176 submission_runner.py:516] Time since start: 12634.51s, Step: 15907, {'train/ctc_loss': 0.257056379160184, 'train/wer': 0.09424459654956492, 'validation/ctc_loss': 0.49250747775705744, 'validation/wer': 0.14754019214985759, 'validation/num_examples': 5348, 'test/ctc_loss': 0.29026623189399314, 'test/wer': 0.09834866857595516, 'test/num_examples': 2472, 'score': 12233.473339557648, 'total_duration': 12634.514830112457, 'accumulated_submission_time': 12233.473339557648, 'accumulated_eval_time': 379.58304166793823, 'accumulated_logging_time': 0.8120191097259521} +I0911 18:13:53.787334 140516691797760 logging_writer.py:48] [15907] accumulated_eval_time=379.583, accumulated_logging_time=0.812019, accumulated_submission_time=12233.5, global_step=15907, preemption_count=0, score=12233.5, test/ctc_loss=0.290266, test/num_examples=2472, test/wer=0.0983487, total_duration=12634.5, train/ctc_loss=0.257056, train/wer=0.0942446, validation/ctc_loss=0.492507, validation/num_examples=5348, validation/wer=0.14754 +I0911 18:14:55.492394 140516683405056 logging_writer.py:48] [16000] global_step=16000, grad_norm=0.499999, loss=1.42268 +I0911 18:14:55.495928 140540003882176 submission.py:307] 16000) loss = 1.423, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 18:23:01.405701 140516691797760 logging_writer.py:48] [16500] global_step=16500, grad_norm=0.499999, loss=1.35344 +I0911 18:23:01.412752 140540003882176 submission.py:307] 16500) loss = 1.353, grad_norm = 0.500 +I0911 18:26:57.859417 140516683405056 logging_writer.py:48] [17000] global_step=17000, grad_norm=0.499999, loss=1.31572 +I0911 18:26:57.863276 140540003882176 submission.py:307] 17000) loss = 1.316, grad_norm = 0.500 +I0911 18:35:03.338078 140516691797760 logging_writer.py:48] [17500] global_step=17500, grad_norm=0.499999, loss=1.3213 +I0911 18:35:03.341898 140540003882176 submission.py:307] 17500) loss = 1.321, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 18:39:01.309153 140516691797760 logging_writer.py:48] [18000] global_step=18000, grad_norm=0.499999, loss=1.32756 +I0911 18:39:01.315555 140540003882176 submission.py:307] 18000) loss = 1.328, grad_norm = 0.500 +I0911 18:43:03.999488 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 18:43:19.473220 140540003882176 spec.py:346] Evaluating on the validation split. +I0911 18:43:39.140114 140540003882176 spec.py:363] Evaluating on the test split. +I0911 18:43:49.299387 140540003882176 submission_runner.py:516] Time since start: 14430.20s, Step: 18300, {'train/ctc_loss': 0.24254760535750525, 'train/wer': 0.08956988318614643, 'validation/ctc_loss': 0.47479960552918427, 'validation/wer': 0.14228745232462706, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2783933640880057, 'test/wer': 0.0924786220624378, 'test/num_examples': 2472, 'score': 13980.496501922607, 'total_duration': 14430.19681930542, 'accumulated_submission_time': 13980.496501922607, 'accumulated_eval_time': 424.8829026222229, 'accumulated_logging_time': 0.9918942451477051} +I0911 18:43:49.598497 140516691797760 logging_writer.py:48] [18300] accumulated_eval_time=424.883, accumulated_logging_time=0.991894, accumulated_submission_time=13980.5, global_step=18300, preemption_count=0, score=13980.5, test/ctc_loss=0.278393, test/num_examples=2472, test/wer=0.0924786, total_duration=14430.2, train/ctc_loss=0.242548, train/wer=0.0895699, validation/ctc_loss=0.4748, validation/num_examples=5348, validation/wer=0.142287 +I0911 18:47:27.601359 140516683405056 logging_writer.py:48] [18500] global_step=18500, grad_norm=0.499999, loss=1.31863 +I0911 18:47:27.605091 140540003882176 submission.py:307] 18500) loss = 1.319, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 18:51:50.268883 140516691797760 logging_writer.py:48] [19000] global_step=19000, grad_norm=0.499999, loss=1.29776 +I0911 18:51:50.275223 140540003882176 submission.py:307] 19000) loss = 1.298, grad_norm = 0.500 +I0911 18:59:21.406140 140516683405056 logging_writer.py:48] [19500] global_step=19500, grad_norm=0.461211, loss=1.28052 +I0911 18:59:21.410103 140540003882176 submission.py:307] 19500) loss = 1.281, grad_norm = 0.461 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 19:03:55.670095 140516691797760 logging_writer.py:48] [20000] global_step=20000, grad_norm=0.499999, loss=1.27489 +I0911 19:03:55.676738 140540003882176 submission.py:307] 20000) loss = 1.275, grad_norm = 0.500 +I0911 19:11:14.385066 140516683405056 logging_writer.py:48] [20500] global_step=20500, grad_norm=0.499999, loss=1.35196 +I0911 19:11:14.388815 140540003882176 submission.py:307] 20500) loss = 1.352, grad_norm = 0.500 +I0911 19:12:59.368742 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 19:13:13.827678 140540003882176 spec.py:346] Evaluating on the validation split. +I0911 19:13:33.882647 140540003882176 spec.py:363] Evaluating on the test split. +I0911 19:13:44.123497 140540003882176 submission_runner.py:516] Time since start: 16225.02s, Step: 20592, {'train/ctc_loss': 0.22380513617838058, 'train/wer': 0.08316239500788115, 'validation/ctc_loss': 0.45205096765433495, 'validation/wer': 0.1334234538695505, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2681596897380476, 'test/wer': 0.08746166189344545, 'test/num_examples': 2472, 'score': 15726.650465726852, 'total_duration': 16225.020930051804, 'accumulated_submission_time': 15726.650465726852, 'accumulated_eval_time': 469.6380949020386, 'accumulated_logging_time': 1.3005595207214355} +I0911 19:13:44.282414 140516691797760 logging_writer.py:48] [20592] accumulated_eval_time=469.638, accumulated_logging_time=1.30056, accumulated_submission_time=15726.7, global_step=20592, preemption_count=0, score=15726.7, test/ctc_loss=0.26816, test/num_examples=2472, test/wer=0.0874617, total_duration=16225, train/ctc_loss=0.223805, train/wer=0.0831624, validation/ctc_loss=0.452051, validation/num_examples=5348, validation/wer=0.133423 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 19:16:54.355301 140516691797760 logging_writer.py:48] [21000] global_step=21000, grad_norm=0.499999, loss=1.374 +I0911 19:16:54.361958 140540003882176 submission.py:307] 21000) loss = 1.374, grad_norm = 0.500 +I0911 19:24:00.060292 140516683405056 logging_writer.py:48] [21500] global_step=21500, grad_norm=0.488106, loss=1.23421 +I0911 19:24:00.064078 140540003882176 submission.py:307] 21500) loss = 1.234, grad_norm = 0.488 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 19:29:07.894409 140516691797760 logging_writer.py:48] [22000] global_step=22000, grad_norm=0.499999, loss=1.2677 +I0911 19:29:07.901156 140540003882176 submission.py:307] 22000) loss = 1.268, grad_norm = 0.500 +I0911 19:35:48.008638 140516683405056 logging_writer.py:48] [22500] global_step=22500, grad_norm=0.499999, loss=1.34188 +I0911 19:35:48.012727 140540003882176 submission.py:307] 22500) loss = 1.342, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 19:41:19.125346 140516691797760 logging_writer.py:48] [23000] global_step=23000, grad_norm=0.499999, loss=1.30297 +I0911 19:41:19.133889 140540003882176 submission.py:307] 23000) loss = 1.303, grad_norm = 0.500 +I0911 19:42:53.733566 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 19:43:09.044112 140540003882176 spec.py:346] Evaluating on the validation split. +I0911 19:43:28.681866 140540003882176 spec.py:363] Evaluating on the test split. +I0911 19:43:38.889744 140540003882176 submission_runner.py:516] Time since start: 18019.79s, Step: 23169, {'train/ctc_loss': 0.21370617326489888, 'train/wer': 0.07796407056334075, 'validation/ctc_loss': 0.45444474269138035, 'validation/wer': 0.13255443441317047, 'validation/num_examples': 5348, 'test/ctc_loss': 0.26259806580263245, 'test/wer': 0.0869741839822883, 'test/num_examples': 2472, 'score': 17472.34481048584, 'total_duration': 18019.78716826439, 'accumulated_submission_time': 17472.34481048584, 'accumulated_eval_time': 514.7942316532135, 'accumulated_logging_time': 1.469043493270874} +I0911 19:43:39.240080 140516691797760 logging_writer.py:48] [23169] accumulated_eval_time=514.794, accumulated_logging_time=1.46904, accumulated_submission_time=17472.3, global_step=23169, preemption_count=0, score=17472.3, test/ctc_loss=0.262598, test/num_examples=2472, test/wer=0.0869742, total_duration=18019.8, train/ctc_loss=0.213706, train/wer=0.0779641, validation/ctc_loss=0.454445, validation/num_examples=5348, validation/wer=0.132554 +I0911 19:48:31.951032 140516683405056 logging_writer.py:48] [23500] global_step=23500, grad_norm=0.499999, loss=1.30203 +I0911 19:48:31.954765 140540003882176 submission.py:307] 23500) loss = 1.302, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 19:54:16.626210 140516691797760 logging_writer.py:48] [24000] global_step=24000, grad_norm=0.422847, loss=1.25286 +I0911 19:54:16.632886 140540003882176 submission.py:307] 24000) loss = 1.253, grad_norm = 0.423 +I0911 20:00:13.851908 140516683405056 logging_writer.py:48] [24500] global_step=24500, grad_norm=0.499999, loss=1.24047 +I0911 20:00:13.855665 140540003882176 submission.py:307] 24500) loss = 1.240, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 20:06:19.189130 140516691797760 logging_writer.py:48] [25000] global_step=25000, grad_norm=0.499999, loss=1.27188 +I0911 20:06:19.197629 140540003882176 submission.py:307] 25000) loss = 1.272, grad_norm = 0.500 +I0911 20:11:58.614310 140516683405056 logging_writer.py:48] [25500] global_step=25500, grad_norm=0.499999, loss=1.19087 +I0911 20:11:58.617935 140540003882176 submission.py:307] 25500) loss = 1.191, grad_norm = 0.500 +I0911 20:12:47.787536 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 20:13:04.080148 140540003882176 spec.py:346] Evaluating on the validation split. +I0911 20:13:23.828732 140540003882176 spec.py:363] Evaluating on the test split. +I0911 20:13:33.902662 140540003882176 submission_runner.py:516] Time since start: 19814.80s, Step: 25545, {'train/ctc_loss': 0.20123739314161537, 'train/wer': 0.07570768466737202, 'validation/ctc_loss': 0.4330134438730661, 'validation/wer': 0.12778448317481775, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2519238136331376, 'test/wer': 0.08297280279487336, 'test/num_examples': 2472, 'score': 19217.717973709106, 'total_duration': 19814.8000998497, 'accumulated_submission_time': 19217.717973709106, 'accumulated_eval_time': 560.9092576503754, 'accumulated_logging_time': 1.8289296627044678} +I0911 20:13:34.132802 140516691797760 logging_writer.py:48] [25545] accumulated_eval_time=560.909, accumulated_logging_time=1.82893, accumulated_submission_time=19217.7, global_step=25545, preemption_count=0, score=19217.7, test/ctc_loss=0.251924, test/num_examples=2472, test/wer=0.0829728, total_duration=19814.8, train/ctc_loss=0.201237, train/wer=0.0757077, validation/ctc_loss=0.433013, validation/num_examples=5348, validation/wer=0.127784 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 20:19:07.540869 140516691797760 logging_writer.py:48] [26000] global_step=26000, grad_norm=0.498073, loss=1.26352 +I0911 20:19:07.557080 140540003882176 submission.py:307] 26000) loss = 1.264, grad_norm = 0.498 +I0911 20:24:41.224660 140516683405056 logging_writer.py:48] [26500] global_step=26500, grad_norm=0.499999, loss=1.2804 +I0911 20:24:41.228431 140540003882176 submission.py:307] 26500) loss = 1.280, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 20:31:23.545976 140516691797760 logging_writer.py:48] [27000] global_step=27000, grad_norm=0.499999, loss=1.28349 +I0911 20:31:23.553042 140540003882176 submission.py:307] 27000) loss = 1.283, grad_norm = 0.500 +I0911 20:36:31.787814 140516683405056 logging_writer.py:48] [27500] global_step=27500, grad_norm=0.499999, loss=1.18089 +I0911 20:36:31.791450 140540003882176 submission.py:307] 27500) loss = 1.181, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 20:42:42.939997 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +I0911 20:42:56.858480 140540003882176 spec.py:346] Evaluating on the validation split. +I0911 20:43:16.471050 140540003882176 spec.py:363] Evaluating on the test split. +I0911 20:43:26.755070 140540003882176 submission_runner.py:516] Time since start: 21607.65s, Step: 27875, {'train/ctc_loss': 0.1893225379236285, 'train/wer': 0.07128128171355774, 'validation/ctc_loss': 0.4232125982645168, 'validation/wer': 0.12426012649060976, 'validation/num_examples': 5348, 'test/ctc_loss': 0.24337820181947223, 'test/wer': 0.07840269737777507, 'test/num_examples': 2472, 'score': 20962.9728038311, 'total_duration': 21607.652501821518, 'accumulated_submission_time': 20962.9728038311, 'accumulated_eval_time': 604.724237203598, 'accumulated_logging_time': 2.0684447288513184} +I0911 20:43:26.908110 140516691797760 logging_writer.py:48] [27875] accumulated_eval_time=604.724, accumulated_logging_time=2.06844, accumulated_submission_time=20963, global_step=27875, preemption_count=0, score=20963, test/ctc_loss=0.243378, test/num_examples=2472, test/wer=0.0784027, total_duration=21607.7, train/ctc_loss=0.189323, train/wer=0.0712813, validation/ctc_loss=0.423213, validation/num_examples=5348, validation/wer=0.12426 +I0911 20:44:23.125929 140516683405056 logging_writer.py:48] [28000] global_step=28000, grad_norm=0.499999, loss=1.19513 +I0911 20:44:23.129385 140540003882176 submission.py:307] 28000) loss = 1.195, grad_norm = 0.500 +I0911 20:49:32.663340 140516691797760 logging_writer.py:48] [28500] global_step=28500, grad_norm=0.499999, loss=1.21304 +I0911 20:49:32.666934 140540003882176 submission.py:307] 28500) loss = 1.213, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 20:56:44.542186 140516691797760 logging_writer.py:48] [29000] global_step=29000, grad_norm=0.499999, loss=1.12379 +I0911 20:56:44.550928 140540003882176 submission.py:307] 29000) loss = 1.124, grad_norm = 0.500 +I0911 21:01:25.141401 140516683405056 logging_writer.py:48] [29500] global_step=29500, grad_norm=0.427762, loss=1.23749 +I0911 21:01:25.145085 140540003882176 submission.py:307] 29500) loss = 1.237, grad_norm = 0.428 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 21:08:56.746591 140516691797760 logging_writer.py:48] [30000] global_step=30000, grad_norm=0.499999, loss=1.30069 +I0911 21:08:56.753441 140540003882176 submission.py:307] 30000) loss = 1.301, grad_norm = 0.500 +I0911 21:12:37.054292 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 21:12:52.413242 140540003882176 spec.py:346] Evaluating on the validation split. +I0911 21:13:12.285462 140540003882176 spec.py:363] Evaluating on the test split. +I0911 21:13:22.458730 140540003882176 submission_runner.py:516] Time since start: 23403.36s, Step: 30432, {'train/ctc_loss': 0.18024398802678754, 'train/wer': 0.06785891650292575, 'validation/ctc_loss': 0.4170667001802039, 'validation/wer': 0.12083232752377734, 'validation/num_examples': 5348, 'test/ctc_loss': 0.23676943371346537, 'test/wer': 0.07620904677756789, 'test/num_examples': 2472, 'score': 22708.710742235184, 'total_duration': 23403.356159448624, 'accumulated_submission_time': 22708.710742235184, 'accumulated_eval_time': 650.1285529136658, 'accumulated_logging_time': 2.2313308715820312} +I0911 21:13:22.793680 140516691797760 logging_writer.py:48] [30432] accumulated_eval_time=650.129, accumulated_logging_time=2.23133, accumulated_submission_time=22708.7, global_step=30432, preemption_count=0, score=22708.7, test/ctc_loss=0.236769, test/num_examples=2472, test/wer=0.076209, total_duration=23403.4, train/ctc_loss=0.180244, train/wer=0.0678589, validation/ctc_loss=0.417067, validation/num_examples=5348, validation/wer=0.120832 +I0911 21:14:13.643963 140516683405056 logging_writer.py:48] [30500] global_step=30500, grad_norm=0.499999, loss=1.24928 +I0911 21:14:13.647164 140540003882176 submission.py:307] 30500) loss = 1.249, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 21:21:51.218627 140516691797760 logging_writer.py:48] [31000] global_step=31000, grad_norm=0.499999, loss=1.29396 +I0911 21:21:51.225528 140540003882176 submission.py:307] 31000) loss = 1.294, grad_norm = 0.500 +I0911 21:26:09.275548 140516683405056 logging_writer.py:48] [31500] global_step=31500, grad_norm=0.464595, loss=1.22751 +I0911 21:26:09.279235 140540003882176 submission.py:307] 31500) loss = 1.228, grad_norm = 0.465 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 21:33:55.635967 140516691797760 logging_writer.py:48] [32000] global_step=32000, grad_norm=0.499999, loss=1.22454 +I0911 21:33:55.642977 140540003882176 submission.py:307] 32000) loss = 1.225, grad_norm = 0.500 +I0911 21:38:07.108667 140516683405056 logging_writer.py:48] [32500] global_step=32500, grad_norm=0.499999, loss=1.14428 +I0911 21:38:07.112375 140540003882176 submission.py:307] 32500) loss = 1.144, grad_norm = 0.500 +I0911 21:42:31.750348 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 21:42:47.797314 140540003882176 spec.py:346] Evaluating on the validation split. +I0911 21:43:07.244874 140540003882176 spec.py:363] Evaluating on the test split. +I0911 21:43:17.327332 140540003882176 submission_runner.py:516] Time since start: 25198.22s, Step: 32790, {'train/ctc_loss': 0.16864312840340256, 'train/wer': 0.06429080387796084, 'validation/ctc_loss': 0.4079771833245429, 'validation/wer': 0.118881861632791, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2304708922753081, 'test/wer': 0.07517315621635895, 'test/num_examples': 2472, 'score': 24454.32293319702, 'total_duration': 25198.224741458893, 'accumulated_submission_time': 24454.32293319702, 'accumulated_eval_time': 695.7054653167725, 'accumulated_logging_time': 2.575826644897461} +I0911 21:43:17.391345 140516691797760 logging_writer.py:48] [32790] accumulated_eval_time=695.705, accumulated_logging_time=2.57583, accumulated_submission_time=24454.3, global_step=32790, preemption_count=0, score=24454.3, test/ctc_loss=0.230471, test/num_examples=2472, test/wer=0.0751732, total_duration=25198.2, train/ctc_loss=0.168643, train/wer=0.0642908, validation/ctc_loss=0.407977, validation/num_examples=5348, validation/wer=0.118882 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 21:46:43.783098 140516691797760 logging_writer.py:48] [33000] global_step=33000, grad_norm=0.499999, loss=1.11679 +I0911 21:46:43.789541 140540003882176 submission.py:307] 33000) loss = 1.117, grad_norm = 0.500 +I0911 21:50:54.536852 140516683405056 logging_writer.py:48] [33500] global_step=33500, grad_norm=0.434205, loss=1.16856 +I0911 21:50:54.540523 140540003882176 submission.py:307] 33500) loss = 1.169, grad_norm = 0.434 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 21:58:59.811427 140516691797760 logging_writer.py:48] [34000] global_step=34000, grad_norm=0.42842, loss=1.07323 +I0911 21:58:59.818272 140540003882176 submission.py:307] 34000) loss = 1.073, grad_norm = 0.428 +I0911 22:02:54.715799 140516683405056 logging_writer.py:48] [34500] global_step=34500, grad_norm=0.499999, loss=1.1549 +I0911 22:02:54.719647 140540003882176 submission.py:307] 34500) loss = 1.155, grad_norm = 0.500 +I0911 22:10:58.680139 140516691797760 logging_writer.py:48] [35000] global_step=35000, grad_norm=0.499999, loss=1.16888 +I0911 22:10:58.683857 140540003882176 submission.py:307] 35000) loss = 1.169, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 22:12:26.536139 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +I0911 22:12:40.374633 140540003882176 spec.py:346] Evaluating on the validation split. +I0911 22:12:59.866880 140540003882176 spec.py:363] Evaluating on the test split. +I0911 22:13:10.091265 140540003882176 submission_runner.py:516] Time since start: 26990.99s, Step: 35167, {'train/ctc_loss': 0.16332430779865387, 'train/wer': 0.061845486148597584, 'validation/ctc_loss': 0.40148898801675204, 'validation/wer': 0.11561821078549703, 'validation/num_examples': 5348, 'test/ctc_loss': 0.22793602611458805, 'test/wer': 0.07289825929762557, 'test/num_examples': 2472, 'score': 26199.73352622986, 'total_duration': 26990.98868870735, 'accumulated_submission_time': 26199.73352622986, 'accumulated_eval_time': 739.260491847992, 'accumulated_logging_time': 2.6493473052978516} +I0911 22:13:10.412585 140516691797760 logging_writer.py:48] [35167] accumulated_eval_time=739.26, accumulated_logging_time=2.64935, accumulated_submission_time=26199.7, global_step=35167, preemption_count=0, score=26199.7, test/ctc_loss=0.227936, test/num_examples=2472, test/wer=0.0728983, total_duration=26991, train/ctc_loss=0.163324, train/wer=0.0618455, validation/ctc_loss=0.401489, validation/num_examples=5348, validation/wer=0.115618 +I0911 22:15:59.583776 140516683405056 logging_writer.py:48] [35500] global_step=35500, grad_norm=0.499472, loss=1.1197 +I0911 22:15:59.594679 140540003882176 submission.py:307] 35500) loss = 1.120, grad_norm = 0.499 +I0911 22:23:46.925575 140516691797760 logging_writer.py:48] [36000] global_step=36000, grad_norm=0.499999, loss=1.12788 +I0911 22:23:46.929257 140540003882176 submission.py:307] 36000) loss = 1.128, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 22:28:05.247055 140516691797760 logging_writer.py:48] [36500] global_step=36500, grad_norm=0.491212, loss=1.08695 +I0911 22:28:05.255455 140540003882176 submission.py:307] 36500) loss = 1.087, grad_norm = 0.491 +I0911 22:35:30.668827 140516683405056 logging_writer.py:48] [37000] global_step=37000, grad_norm=0.499999, loss=1.15954 +I0911 22:35:30.672581 140540003882176 submission.py:307] 37000) loss = 1.160, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 22:40:11.592837 140516691797760 logging_writer.py:48] [37500] global_step=37500, grad_norm=0.499999, loss=1.18316 +I0911 22:40:11.601159 140540003882176 submission.py:307] 37500) loss = 1.183, grad_norm = 0.500 +I0911 22:42:20.114426 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 22:42:36.050319 140540003882176 spec.py:346] Evaluating on the validation split. +I0911 22:42:55.907489 140540003882176 spec.py:363] Evaluating on the test split. +I0911 22:43:06.134293 140540003882176 submission_runner.py:516] Time since start: 28787.03s, Step: 37696, {'train/ctc_loss': 0.15306217709954, 'train/wer': 0.058520285880854186, 'validation/ctc_loss': 0.39821950503566406, 'validation/wer': 0.11365808912277314, 'validation/num_examples': 5348, 'test/ctc_loss': 0.22283302551777534, 'test/wer': 0.0707046086974184, 'test/num_examples': 2472, 'score': 27945.150611400604, 'total_duration': 28787.031717061996, 'accumulated_submission_time': 27945.150611400604, 'accumulated_eval_time': 785.2804026603699, 'accumulated_logging_time': 2.9802584648132324} +I0911 22:43:06.457927 140516691797760 logging_writer.py:48] [37696] accumulated_eval_time=785.28, accumulated_logging_time=2.98026, accumulated_submission_time=27945.2, global_step=37696, preemption_count=0, score=27945.2, test/ctc_loss=0.222833, test/num_examples=2472, test/wer=0.0707046, total_duration=28787, train/ctc_loss=0.153062, train/wer=0.0585203, validation/ctc_loss=0.39822, validation/num_examples=5348, validation/wer=0.113658 +I0911 22:48:06.956344 140516683405056 logging_writer.py:48] [38000] global_step=38000, grad_norm=0.499999, loss=1.12971 +I0911 22:48:06.959902 140540003882176 submission.py:307] 38000) loss = 1.130, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 22:53:06.648444 140516691797760 logging_writer.py:48] [38500] global_step=38500, grad_norm=0.499999, loss=1.06353 +I0911 22:53:06.655162 140540003882176 submission.py:307] 38500) loss = 1.064, grad_norm = 0.500 +I0911 22:59:54.255245 140516683405056 logging_writer.py:48] [39000] global_step=39000, grad_norm=0.499999, loss=1.16653 +I0911 22:59:54.259070 140540003882176 submission.py:307] 39000) loss = 1.167, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 23:05:08.948867 140516691797760 logging_writer.py:48] [39500] global_step=39500, grad_norm=0.499999, loss=1.12998 +I0911 23:05:08.955539 140540003882176 submission.py:307] 39500) loss = 1.130, grad_norm = 0.500 +I0911 23:11:43.128129 140516683405056 logging_writer.py:48] [40000] global_step=40000, grad_norm=0.499999, loss=1.19382 +I0911 23:11:43.131832 140540003882176 submission.py:307] 40000) loss = 1.194, grad_norm = 0.500 +I0911 23:12:14.624644 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 23:12:28.909391 140540003882176 spec.py:346] Evaluating on the validation split. +I0911 23:12:48.664129 140540003882176 spec.py:363] Evaluating on the test split. +I0911 23:12:58.939248 140540003882176 submission_runner.py:516] Time since start: 30579.84s, Step: 40029, {'train/ctc_loss': 0.1467357326616722, 'train/wer': 0.05645283181827997, 'validation/ctc_loss': 0.38837735615079366, 'validation/wer': 0.1106744556558683, 'validation/num_examples': 5348, 'test/ctc_loss': 0.21621977123524097, 'test/wer': 0.06897812442873683, 'test/num_examples': 2472, 'score': 29690.69269633293, 'total_duration': 30579.836644887924, 'accumulated_submission_time': 29690.69269633293, 'accumulated_eval_time': 829.5948066711426, 'accumulated_logging_time': 3.313499689102173} +I0911 23:12:59.008921 140516691797760 logging_writer.py:48] [40029] accumulated_eval_time=829.595, accumulated_logging_time=3.3135, accumulated_submission_time=29690.7, global_step=40029, preemption_count=0, score=29690.7, test/ctc_loss=0.21622, test/num_examples=2472, test/wer=0.0689781, total_duration=30579.8, train/ctc_loss=0.146736, train/wer=0.0564528, validation/ctc_loss=0.388377, validation/num_examples=5348, validation/wer=0.110674 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 23:18:02.059260 140516691797760 logging_writer.py:48] [40500] global_step=40500, grad_norm=0.488643, loss=1.12346 +I0911 23:18:02.072029 140540003882176 submission.py:307] 40500) loss = 1.123, grad_norm = 0.489 +I0911 23:24:22.767590 140516683405056 logging_writer.py:48] [41000] global_step=41000, grad_norm=0.499999, loss=1.0686 +I0911 23:24:22.771416 140540003882176 submission.py:307] 41000) loss = 1.069, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 23:30:12.614494 140516691797760 logging_writer.py:48] [41500] global_step=41500, grad_norm=0.499999, loss=1.09138 +I0911 23:30:12.620961 140540003882176 submission.py:307] 41500) loss = 1.091, grad_norm = 0.500 +I0911 23:36:09.202648 140516683405056 logging_writer.py:48] [42000] global_step=42000, grad_norm=0.5, loss=1.20214 +I0911 23:36:09.206469 140540003882176 submission.py:307] 42000) loss = 1.202, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 23:42:08.592719 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +I0911 23:42:23.637694 140540003882176 spec.py:346] Evaluating on the validation split. +I0911 23:42:43.457733 140540003882176 spec.py:363] Evaluating on the test split. +I0911 23:42:53.689954 140540003882176 submission_runner.py:516] Time since start: 32374.59s, Step: 42457, {'train/ctc_loss': 0.1373192074511263, 'train/wer': 0.05334895169822728, 'validation/ctc_loss': 0.3837925943716094, 'validation/wer': 0.10904263023222131, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2129698387799213, 'test/wer': 0.0666829159303719, 'test/num_examples': 2472, 'score': 31436.16641187668, 'total_duration': 32374.587361335754, 'accumulated_submission_time': 31436.16641187668, 'accumulated_eval_time': 874.6922862529755, 'accumulated_logging_time': 3.3926544189453125} +I0911 23:42:53.966593 140516691797760 logging_writer.py:48] [42457] accumulated_eval_time=874.692, accumulated_logging_time=3.39265, accumulated_submission_time=31436.2, global_step=42457, preemption_count=0, score=31436.2, test/ctc_loss=0.21297, test/num_examples=2472, test/wer=0.0666829, total_duration=32374.6, train/ctc_loss=0.137319, train/wer=0.053349, validation/ctc_loss=0.383793, validation/num_examples=5348, validation/wer=0.109043 +I0911 23:43:15.155580 140516683405056 logging_writer.py:48] [42500] global_step=42500, grad_norm=0.478564, loss=1.03617 +I0911 23:43:15.158838 140540003882176 submission.py:307] 42500) loss = 1.036, grad_norm = 0.479 +I0911 23:49:06.425388 140516691797760 logging_writer.py:48] [43000] global_step=43000, grad_norm=0.496737, loss=1.07621 +I0911 23:49:06.429068 140540003882176 submission.py:307] 43000) loss = 1.076, grad_norm = 0.497 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0911 23:55:36.205920 140516691797760 logging_writer.py:48] [43500] global_step=43500, grad_norm=0.499999, loss=1.13714 +I0911 23:55:36.212484 140540003882176 submission.py:307] 43500) loss = 1.137, grad_norm = 0.500 +I0912 00:00:55.157982 140516683405056 logging_writer.py:48] [44000] global_step=44000, grad_norm=0.499999, loss=1.12373 +I0912 00:00:55.161903 140540003882176 submission.py:307] 44000) loss = 1.124, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 00:07:46.329214 140516691797760 logging_writer.py:48] [44500] global_step=44500, grad_norm=0.499999, loss=1.11591 +I0912 00:07:46.335756 140540003882176 submission.py:307] 44500) loss = 1.116, grad_norm = 0.500 +I0912 00:12:04.227102 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 00:12:19.728863 140540003882176 spec.py:346] Evaluating on the validation split. +I0912 00:12:39.508233 140540003882176 spec.py:363] Evaluating on the test split. +I0912 00:12:49.742401 140540003882176 submission_runner.py:516] Time since start: 34170.64s, Step: 44951, {'train/ctc_loss': 0.12914611325086248, 'train/wer': 0.05020728521149569, 'validation/ctc_loss': 0.3754650052350186, 'validation/wer': 0.1060300294501038, 'validation/num_examples': 5348, 'test/ctc_loss': 0.20743576198464417, 'test/wer': 0.06680478540816119, 'test/num_examples': 2472, 'score': 33182.07935786247, 'total_duration': 34170.63980221748, 'accumulated_submission_time': 33182.07935786247, 'accumulated_eval_time': 920.2076592445374, 'accumulated_logging_time': 3.678757905960083} +I0912 00:12:50.068136 140516691797760 logging_writer.py:48] [44951] accumulated_eval_time=920.208, accumulated_logging_time=3.67876, accumulated_submission_time=33182.1, global_step=44951, preemption_count=0, score=33182.1, test/ctc_loss=0.207436, test/num_examples=2472, test/wer=0.0668048, total_duration=34170.6, train/ctc_loss=0.129146, train/wer=0.0502073, validation/ctc_loss=0.375465, validation/num_examples=5348, validation/wer=0.10603 +I0912 00:13:32.799895 140516683405056 logging_writer.py:48] [45000] global_step=45000, grad_norm=0.499999, loss=1.07246 +I0912 00:13:32.803316 140540003882176 submission.py:307] 45000) loss = 1.072, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 00:20:38.331431 140516691797760 logging_writer.py:48] [45500] global_step=45500, grad_norm=0.499999, loss=1.14548 +I0912 00:20:38.338599 140540003882176 submission.py:307] 45500) loss = 1.145, grad_norm = 0.500 +I0912 00:25:34.016728 140516683405056 logging_writer.py:48] [46000] global_step=46000, grad_norm=0.483227, loss=1.09578 +I0912 00:25:34.020457 140540003882176 submission.py:307] 46000) loss = 1.096, grad_norm = 0.483 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 00:32:47.971696 140516691797760 logging_writer.py:48] [46500] global_step=46500, grad_norm=0.489097, loss=1.11385 +I0912 00:32:47.978589 140540003882176 submission.py:307] 46500) loss = 1.114, grad_norm = 0.489 +I0912 00:37:24.535373 140516683405056 logging_writer.py:48] [47000] global_step=47000, grad_norm=0.499999, loss=1.14261 +I0912 00:37:24.539232 140540003882176 submission.py:307] 47000) loss = 1.143, grad_norm = 0.500 +I0912 00:42:00.495447 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 00:42:14.987402 140540003882176 spec.py:346] Evaluating on the validation split. +I0912 00:42:34.819376 140540003882176 spec.py:363] Evaluating on the test split. +I0912 00:42:44.989977 140540003882176 submission_runner.py:516] Time since start: 35965.89s, Step: 47271, {'train/ctc_loss': 0.12275029277717649, 'train/wer': 0.04875520912054931, 'validation/ctc_loss': 0.36745415630650996, 'validation/wer': 0.10363539805918988, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2035757851312988, 'test/wer': 0.06390022952084984, 'test/num_examples': 2472, 'score': 34929.891464710236, 'total_duration': 35965.88739991188, 'accumulated_submission_time': 34929.891464710236, 'accumulated_eval_time': 964.7020807266235, 'accumulated_logging_time': 4.014244794845581} +I0912 00:42:45.066491 140516691797760 logging_writer.py:48] [47271] accumulated_eval_time=964.702, accumulated_logging_time=4.01424, accumulated_submission_time=34929.9, global_step=47271, preemption_count=0, score=34929.9, test/ctc_loss=0.203576, test/num_examples=2472, test/wer=0.0639002, total_duration=35965.9, train/ctc_loss=0.12275, train/wer=0.0487552, validation/ctc_loss=0.367454, validation/num_examples=5348, validation/wer=0.103635 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 00:45:38.332887 140516691797760 logging_writer.py:48] [47500] global_step=47500, grad_norm=0.434586, loss=1.06484 +I0912 00:45:38.341110 140540003882176 submission.py:307] 47500) loss = 1.065, grad_norm = 0.435 +I0912 00:50:16.496064 140516683405056 logging_writer.py:48] [48000] global_step=48000, grad_norm=0.499999, loss=1.11089 +I0912 00:50:16.499839 140540003882176 submission.py:307] 48000) loss = 1.111, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 00:57:50.890405 140516691797760 logging_writer.py:48] [48500] global_step=48500, grad_norm=0.499999, loss=1.09575 +I0912 00:57:50.897637 140540003882176 submission.py:307] 48500) loss = 1.096, grad_norm = 0.500 +I0912 01:02:05.325849 140516683405056 logging_writer.py:48] [49000] global_step=49000, grad_norm=0.499999, loss=1.06568 +I0912 01:02:05.329697 140540003882176 submission.py:307] 49000) loss = 1.066, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 01:10:02.324716 140516691797760 logging_writer.py:48] [49500] global_step=49500, grad_norm=0.499999, loss=1.0886 +I0912 01:10:02.331333 140540003882176 submission.py:307] 49500) loss = 1.089, grad_norm = 0.500 +I0912 01:11:54.710160 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 01:12:09.700285 140540003882176 spec.py:346] Evaluating on the validation split. +I0912 01:12:29.484032 140540003882176 spec.py:363] Evaluating on the test split. +I0912 01:12:39.773905 140540003882176 submission_runner.py:516] Time since start: 37760.67s, Step: 49757, {'train/ctc_loss': 0.11352582992288439, 'train/wer': 0.0453112517003865, 'validation/ctc_loss': 0.3629400810132861, 'validation/wer': 0.10128904552696374, 'validation/num_examples': 5348, 'test/ctc_loss': 0.19862260518823796, 'test/wer': 0.06156439786322182, 'test/num_examples': 2472, 'score': 36675.30523777008, 'total_duration': 37760.6712975502, 'accumulated_submission_time': 36675.30523777008, 'accumulated_eval_time': 1009.7656865119934, 'accumulated_logging_time': 4.100290775299072} +I0912 01:12:40.119415 140516691797760 logging_writer.py:48] [49757] accumulated_eval_time=1009.77, accumulated_logging_time=4.10029, accumulated_submission_time=36675.3, global_step=49757, preemption_count=0, score=36675.3, test/ctc_loss=0.198623, test/num_examples=2472, test/wer=0.0615644, total_duration=37760.7, train/ctc_loss=0.113526, train/wer=0.0453113, validation/ctc_loss=0.36294, validation/num_examples=5348, validation/wer=0.101289 +I0912 01:15:07.572027 140516683405056 logging_writer.py:48] [50000] global_step=50000, grad_norm=0.499999, loss=1.05634 +I0912 01:15:07.575605 140540003882176 submission.py:307] 50000) loss = 1.056, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 01:23:08.554352 140516691797760 logging_writer.py:48] [50500] global_step=50500, grad_norm=0.499999, loss=1.05543 +I0912 01:23:08.560933 140540003882176 submission.py:307] 50500) loss = 1.055, grad_norm = 0.500 +I0912 01:27:07.721402 140516683405056 logging_writer.py:48] [51000] global_step=51000, grad_norm=0.499999, loss=0.998811 +I0912 01:27:07.725612 140540003882176 submission.py:307] 51000) loss = 0.999, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 01:35:17.845055 140516691797760 logging_writer.py:48] [51500] global_step=51500, grad_norm=0.499999, loss=1.10889 +I0912 01:35:17.851891 140540003882176 submission.py:307] 51500) loss = 1.109, grad_norm = 0.500 +I0912 01:39:10.405989 140516683405056 logging_writer.py:48] [52000] global_step=52000, grad_norm=0.499999, loss=1.04401 +I0912 01:39:10.409974 140540003882176 submission.py:307] 52000) loss = 1.044, grad_norm = 0.500 +I0912 01:41:51.583162 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 01:42:07.623344 140540003882176 spec.py:346] Evaluating on the validation split. +I0912 01:42:27.258980 140540003882176 spec.py:363] Evaluating on the test split. +I0912 01:42:37.416456 140540003882176 submission_runner.py:516] Time since start: 39558.31s, Step: 52211, {'train/ctc_loss': 0.10484115516978962, 'train/wer': 0.04175933323256969, 'validation/ctc_loss': 0.35476956205576904, 'validation/wer': 0.0994930719837783, 'validation/num_examples': 5348, 'test/ctc_loss': 0.195299028868798, 'test/wer': 0.060873804155749195, 'test/num_examples': 2472, 'score': 38422.19172549248, 'total_duration': 39558.31389641762, 'accumulated_submission_time': 38422.19172549248, 'accumulated_eval_time': 1055.598946094513, 'accumulated_logging_time': 4.455331563949585} +I0912 01:42:37.713718 140516691797760 logging_writer.py:48] [52211] accumulated_eval_time=1055.6, accumulated_logging_time=4.45533, accumulated_submission_time=38422.2, global_step=52211, preemption_count=0, score=38422.2, test/ctc_loss=0.195299, test/num_examples=2472, test/wer=0.0608738, total_duration=39558.3, train/ctc_loss=0.104841, train/wer=0.0417593, validation/ctc_loss=0.35477, validation/num_examples=5348, validation/wer=0.0994931 +I0912 01:47:47.518769 140516683405056 logging_writer.py:48] [52500] global_step=52500, grad_norm=0.499999, loss=0.958897 +I0912 01:47:47.522432 140540003882176 submission.py:307] 52500) loss = 0.959, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 01:52:02.634957 140516691797760 logging_writer.py:48] [53000] global_step=53000, grad_norm=0.499999, loss=1.07811 +I0912 01:52:02.641777 140540003882176 submission.py:307] 53000) loss = 1.078, grad_norm = 0.500 +I0912 01:59:36.139376 140516683405056 logging_writer.py:48] [53500] global_step=53500, grad_norm=0.499999, loss=1.01615 +I0912 01:59:36.142991 140540003882176 submission.py:307] 53500) loss = 1.016, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 02:04:03.148580 140516691797760 logging_writer.py:48] [54000] global_step=54000, grad_norm=0.478527, loss=1.06287 +I0912 02:04:03.155600 140540003882176 submission.py:307] 54000) loss = 1.063, grad_norm = 0.479 +I0912 02:11:23.774476 140516683405056 logging_writer.py:48] [54500] global_step=54500, grad_norm=0.499999, loss=1.01721 +I0912 02:11:23.778255 140540003882176 submission.py:307] 54500) loss = 1.017, grad_norm = 0.500 +I0912 02:11:47.204452 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 02:12:01.721955 140540003882176 spec.py:346] Evaluating on the validation split. +I0912 02:12:21.505484 140540003882176 spec.py:363] Evaluating on the test split. +I0912 02:12:31.822214 140540003882176 submission_runner.py:516] Time since start: 41352.72s, Step: 54520, {'train/ctc_loss': 0.09859529051528783, 'train/wer': 0.03960551033187226, 'validation/ctc_loss': 0.35677667073224334, 'validation/wer': 0.09722396562545262, 'validation/num_examples': 5348, 'test/ctc_loss': 0.19444095939657396, 'test/wer': 0.059858225174171796, 'test/num_examples': 2472, 'score': 40169.08051586151, 'total_duration': 41352.719628334045, 'accumulated_submission_time': 40169.08051586151, 'accumulated_eval_time': 1100.216621875763, 'accumulated_logging_time': 4.762305021286011} +I0912 02:12:31.897939 140516691797760 logging_writer.py:48] [54520] accumulated_eval_time=1100.22, accumulated_logging_time=4.76231, accumulated_submission_time=40169.1, global_step=54520, preemption_count=0, score=40169.1, test/ctc_loss=0.194441, test/num_examples=2472, test/wer=0.0598582, total_duration=41352.7, train/ctc_loss=0.0985953, train/wer=0.0396055, validation/ctc_loss=0.356777, validation/num_examples=5348, validation/wer=0.097224 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 02:16:57.040221 140516691797760 logging_writer.py:48] [55000] global_step=55000, grad_norm=0.487945, loss=0.99928 +I0912 02:16:57.047081 140540003882176 submission.py:307] 55000) loss = 0.999, grad_norm = 0.488 +I0912 02:24:06.666454 140516683405056 logging_writer.py:48] [55500] global_step=55500, grad_norm=0.5, loss=1.062 +I0912 02:24:06.670202 140540003882176 submission.py:307] 55500) loss = 1.062, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 02:29:07.735478 140516691797760 logging_writer.py:48] [56000] global_step=56000, grad_norm=0.499999, loss=0.993388 +I0912 02:29:07.742141 140540003882176 submission.py:307] 56000) loss = 0.993, grad_norm = 0.500 +I0912 02:35:53.314671 140516683405056 logging_writer.py:48] [56500] global_step=56500, grad_norm=0.499999, loss=1.04207 +I0912 02:35:53.318345 140540003882176 submission.py:307] 56500) loss = 1.042, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 02:41:17.749127 140516691797760 logging_writer.py:48] [57000] global_step=57000, grad_norm=0.499999, loss=1.03642 +I0912 02:41:17.755927 140540003882176 submission.py:307] 57000) loss = 1.036, grad_norm = 0.500 +I0912 02:41:41.563340 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 02:41:56.478827 140540003882176 spec.py:346] Evaluating on the validation split. +I0912 02:42:16.113625 140540003882176 spec.py:363] Evaluating on the test split. +I0912 02:42:26.194609 140540003882176 submission_runner.py:516] Time since start: 43147.09s, Step: 57045, {'train/ctc_loss': 0.09290728793208415, 'train/wer': 0.037311338069224625, 'validation/ctc_loss': 0.35056288222699417, 'validation/wer': 0.09520590933230338, 'validation/num_examples': 5348, 'test/ctc_loss': 0.19041340620765856, 'test/wer': 0.058761399874068206, 'test/num_examples': 2472, 'score': 41914.625618219376, 'total_duration': 43147.09202218056, 'accumulated_submission_time': 41914.625618219376, 'accumulated_eval_time': 1144.8478198051453, 'accumulated_logging_time': 4.847512245178223} +I0912 02:42:26.535324 140516691797760 logging_writer.py:48] [57045] accumulated_eval_time=1144.85, accumulated_logging_time=4.84751, accumulated_submission_time=41914.6, global_step=57045, preemption_count=0, score=41914.6, test/ctc_loss=0.190413, test/num_examples=2472, test/wer=0.0587614, total_duration=43147.1, train/ctc_loss=0.0929073, train/wer=0.0373113, validation/ctc_loss=0.350563, validation/num_examples=5348, validation/wer=0.0952059 +I0912 02:48:41.814487 140516683405056 logging_writer.py:48] [57500] global_step=57500, grad_norm=0.5, loss=1.00292 +I0912 02:48:41.818120 140540003882176 submission.py:307] 57500) loss = 1.003, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 02:54:20.437143 140516691797760 logging_writer.py:48] [58000] global_step=58000, grad_norm=0.499999, loss=0.98764 +I0912 02:54:20.444333 140540003882176 submission.py:307] 58000) loss = 0.988, grad_norm = 0.500 +I0912 03:00:27.829065 140516683405056 logging_writer.py:48] [58500] global_step=58500, grad_norm=0.499999, loss=1.02411 +I0912 03:00:27.832867 140540003882176 submission.py:307] 58500) loss = 1.024, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 03:06:26.792033 140516691797760 logging_writer.py:48] [59000] global_step=59000, grad_norm=0.499999, loss=0.987043 +I0912 03:06:26.799525 140540003882176 submission.py:307] 59000) loss = 0.987, grad_norm = 0.500 +I0912 03:11:35.878545 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 03:11:51.861209 140540003882176 spec.py:346] Evaluating on the validation split. +I0912 03:12:11.495185 140540003882176 spec.py:363] Evaluating on the test split. +I0912 03:12:21.749212 140540003882176 submission_runner.py:516] Time since start: 44942.65s, Step: 59464, {'train/ctc_loss': 0.08657633160767436, 'train/wer': 0.03479044760650357, 'validation/ctc_loss': 0.34359940491887686, 'validation/wer': 0.09352580504996862, 'validation/num_examples': 5348, 'test/ctc_loss': 0.18565676112168528, 'test/wer': 0.057177096662807465, 'test/num_examples': 2472, 'score': 43659.648621320724, 'total_duration': 44942.64661693573, 'accumulated_submission_time': 43659.648621320724, 'accumulated_eval_time': 1190.7182912826538, 'accumulated_logging_time': 5.198246240615845} +I0912 03:12:22.070369 140516691797760 logging_writer.py:48] [59464] accumulated_eval_time=1190.72, accumulated_logging_time=5.19825, accumulated_submission_time=43659.6, global_step=59464, preemption_count=0, score=43659.6, test/ctc_loss=0.185657, test/num_examples=2472, test/wer=0.0571771, total_duration=44942.6, train/ctc_loss=0.0865763, train/wer=0.0347904, validation/ctc_loss=0.343599, validation/num_examples=5348, validation/wer=0.0935258 +I0912 03:12:56.278245 140516683405056 logging_writer.py:48] [59500] global_step=59500, grad_norm=0.499999, loss=0.946845 +I0912 03:12:56.281462 140540003882176 submission.py:307] 59500) loss = 0.947, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 03:19:13.948460 140516691797760 logging_writer.py:48] [60000] global_step=60000, grad_norm=0.499999, loss=1.02942 +I0912 03:19:13.955348 140540003882176 submission.py:307] 60000) loss = 1.029, grad_norm = 0.500 +I0912 03:24:47.465683 140516683405056 logging_writer.py:48] [60500] global_step=60500, grad_norm=0.5, loss=0.982074 +I0912 03:24:47.469432 140540003882176 submission.py:307] 60500) loss = 0.982, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 03:31:23.208467 140516691797760 logging_writer.py:48] [61000] global_step=61000, grad_norm=0.401541, loss=0.919588 +I0912 03:31:23.215294 140540003882176 submission.py:307] 61000) loss = 0.920, grad_norm = 0.402 +I0912 03:36:34.843818 140516683405056 logging_writer.py:48] [61500] global_step=61500, grad_norm=0.499999, loss=0.9495 +I0912 03:36:34.847477 140540003882176 submission.py:307] 61500) loss = 0.949, grad_norm = 0.500 +I0912 03:41:30.104533 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 03:41:44.415565 140540003882176 spec.py:346] Evaluating on the validation split. +I0912 03:42:04.086154 140540003882176 spec.py:363] Evaluating on the test split. +I0912 03:42:14.242615 140540003882176 submission_runner.py:516] Time since start: 46735.14s, Step: 61769, {'train/ctc_loss': 0.08205772017477846, 'train/wer': 0.032782372120139056, 'validation/ctc_loss': 0.3425308420688291, 'validation/wer': 0.09267609713706368, 'validation/num_examples': 5348, 'test/ctc_loss': 0.18184947163929932, 'test/wer': 0.05644587979607174, 'test/num_examples': 2472, 'score': 45405.07888031006, 'total_duration': 46735.140020132065, 'accumulated_submission_time': 45405.07888031006, 'accumulated_eval_time': 1234.8561778068542, 'accumulated_logging_time': 5.529304265975952} +I0912 03:42:14.304637 140516691797760 logging_writer.py:48] [61769] accumulated_eval_time=1234.86, accumulated_logging_time=5.5293, accumulated_submission_time=45405.1, global_step=61769, preemption_count=0, score=45405.1, test/ctc_loss=0.181849, test/num_examples=2472, test/wer=0.0564459, total_duration=46735.1, train/ctc_loss=0.0820577, train/wer=0.0327824, validation/ctc_loss=0.342531, validation/num_examples=5348, validation/wer=0.0926761 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 03:44:13.339699 140516691797760 logging_writer.py:48] [62000] global_step=62000, grad_norm=0.499999, loss=0.947553 +I0912 03:44:13.345919 140540003882176 submission.py:307] 62000) loss = 0.948, grad_norm = 0.500 +I0912 03:49:26.192633 140516683405056 logging_writer.py:48] [62500] global_step=62500, grad_norm=0.5, loss=1.00196 +I0912 03:49:26.196299 140540003882176 submission.py:307] 62500) loss = 1.002, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 03:56:31.689496 140516691797760 logging_writer.py:48] [63000] global_step=63000, grad_norm=0.499999, loss=0.912381 +I0912 03:56:31.696215 140540003882176 submission.py:307] 63000) loss = 0.912, grad_norm = 0.500 +I0912 04:01:17.282968 140516683405056 logging_writer.py:48] [63500] global_step=63500, grad_norm=0.499999, loss=0.967397 +I0912 04:01:17.286780 140540003882176 submission.py:307] 63500) loss = 0.967, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 04:08:45.151777 140516691797760 logging_writer.py:48] [64000] global_step=64000, grad_norm=0.499999, loss=0.944746 +I0912 04:08:45.158824 140540003882176 submission.py:307] 64000) loss = 0.945, grad_norm = 0.500 +I0912 04:11:23.992232 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 04:11:39.220004 140540003882176 spec.py:346] Evaluating on the validation split. +I0912 04:11:58.989456 140540003882176 spec.py:363] Evaluating on the test split. +I0912 04:12:09.163334 140540003882176 submission_runner.py:516] Time since start: 48530.06s, Step: 64336, {'train/ctc_loss': 0.07784306984796725, 'train/wer': 0.03155701422926608, 'validation/ctc_loss': 0.3417285430951125, 'validation/wer': 0.09078356587650267, 'validation/num_examples': 5348, 'test/ctc_loss': 0.18160254290599392, 'test/wer': 0.05478033026628481, 'test/num_examples': 2472, 'score': 47150.370564222336, 'total_duration': 48530.060735702515, 'accumulated_submission_time': 47150.370564222336, 'accumulated_eval_time': 1280.0275931358337, 'accumulated_logging_time': 5.600770711898804} +I0912 04:12:09.467011 140516691797760 logging_writer.py:48] [64336] accumulated_eval_time=1280.03, accumulated_logging_time=5.60077, accumulated_submission_time=47150.4, global_step=64336, preemption_count=0, score=47150.4, test/ctc_loss=0.181603, test/num_examples=2472, test/wer=0.0547803, total_duration=48530.1, train/ctc_loss=0.0778431, train/wer=0.031557, validation/ctc_loss=0.341729, validation/num_examples=5348, validation/wer=0.0907836 +I0912 04:14:08.547021 140516683405056 logging_writer.py:48] [64500] global_step=64500, grad_norm=0.488113, loss=1.0051 +I0912 04:14:08.550258 140540003882176 submission.py:307] 64500) loss = 1.005, grad_norm = 0.488 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 04:21:42.459164 140516691797760 logging_writer.py:48] [65000] global_step=65000, grad_norm=0.499999, loss=0.994151 +I0912 04:21:42.466048 140540003882176 submission.py:307] 65000) loss = 0.994, grad_norm = 0.500 +I0912 04:26:05.367425 140516683405056 logging_writer.py:48] [65500] global_step=65500, grad_norm=0.499999, loss=0.958848 +I0912 04:26:05.370943 140540003882176 submission.py:307] 65500) loss = 0.959, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 04:33:49.588041 140516691797760 logging_writer.py:48] [66000] global_step=66000, grad_norm=0.499999, loss=1.01925 +I0912 04:33:49.594969 140540003882176 submission.py:307] 66000) loss = 1.019, grad_norm = 0.500 +I0912 04:38:01.942724 140516683405056 logging_writer.py:48] [66500] global_step=66500, grad_norm=0.5, loss=0.954274 +I0912 04:38:01.946416 140540003882176 submission.py:307] 66500) loss = 0.954, grad_norm = 0.500 +I0912 04:41:20.331890 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 04:41:35.921037 140540003882176 spec.py:346] Evaluating on the validation split. +I0912 04:41:55.458321 140540003882176 spec.py:363] Evaluating on the test split. +I0912 04:42:05.710229 140540003882176 submission_runner.py:516] Time since start: 50326.61s, Step: 66723, {'train/ctc_loss': 0.07447617856025503, 'train/wer': 0.030320860233627706, 'validation/ctc_loss': 0.33777348459162143, 'validation/wer': 0.09003041568097331, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17920377423543454, 'test/wer': 0.054800641845916355, 'test/num_examples': 2472, 'score': 48896.60098361969, 'total_duration': 50326.60763835907, 'accumulated_submission_time': 48896.60098361969, 'accumulated_eval_time': 1325.4059286117554, 'accumulated_logging_time': 5.914076328277588} +I0912 04:42:06.001120 140516691797760 logging_writer.py:48] [66723] accumulated_eval_time=1325.41, accumulated_logging_time=5.91408, accumulated_submission_time=48896.6, global_step=66723, preemption_count=0, score=48896.6, test/ctc_loss=0.179204, test/num_examples=2472, test/wer=0.0548006, total_duration=50326.6, train/ctc_loss=0.0744762, train/wer=0.0303209, validation/ctc_loss=0.337773, validation/num_examples=5348, validation/wer=0.0900304 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 04:46:36.535292 140516691797760 logging_writer.py:48] [67000] global_step=67000, grad_norm=0.485116, loss=0.916633 +I0912 04:46:36.545706 140540003882176 submission.py:307] 67000) loss = 0.917, grad_norm = 0.485 +I0912 04:50:51.485169 140516683405056 logging_writer.py:48] [67500] global_step=67500, grad_norm=0.5, loss=0.982497 +I0912 04:50:51.488843 140540003882176 submission.py:307] 67500) loss = 0.982, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 04:58:49.771888 140516691797760 logging_writer.py:48] [68000] global_step=68000, grad_norm=0.499999, loss=0.995658 +I0912 04:58:49.778767 140540003882176 submission.py:307] 68000) loss = 0.996, grad_norm = 0.500 +I0912 05:02:46.216327 140516683405056 logging_writer.py:48] [68500] global_step=68500, grad_norm=0.469277, loss=0.916948 +I0912 05:02:46.220165 140540003882176 submission.py:307] 68500) loss = 0.917, grad_norm = 0.469 +I0912 05:10:52.145920 140516691797760 logging_writer.py:48] [69000] global_step=69000, grad_norm=0.5, loss=0.879451 +I0912 05:10:52.149612 140540003882176 submission.py:307] 69000) loss = 0.879, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 05:11:14.032826 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +I0912 05:11:27.764314 140540003882176 spec.py:346] Evaluating on the validation split. +I0912 05:11:47.638767 140540003882176 spec.py:363] Evaluating on the test split. +I0912 05:11:57.723797 140540003882176 submission_runner.py:516] Time since start: 52118.62s, Step: 69036, {'train/ctc_loss': 0.07208353107243116, 'train/wer': 0.029090104290372034, 'validation/ctc_loss': 0.3336585107902602, 'validation/wer': 0.08863998455076522, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17827939504161558, 'test/wer': 0.05402880181991754, 'test/num_examples': 2472, 'score': 50642.02622580528, 'total_duration': 52118.62122583389, 'accumulated_submission_time': 50642.02622580528, 'accumulated_eval_time': 1369.0967593193054, 'accumulated_logging_time': 6.2146546840667725} +I0912 05:11:57.784540 140516691797760 logging_writer.py:48] [69036] accumulated_eval_time=1369.1, accumulated_logging_time=6.21465, accumulated_submission_time=50642, global_step=69036, preemption_count=0, score=50642, test/ctc_loss=0.178279, test/num_examples=2472, test/wer=0.0540288, total_duration=52118.6, train/ctc_loss=0.0720835, train/wer=0.0290901, validation/ctc_loss=0.333659, validation/num_examples=5348, validation/wer=0.08864 +I0912 05:15:45.209374 140516683405056 logging_writer.py:48] [69500] global_step=69500, grad_norm=0.499999, loss=0.933899 +I0912 05:15:45.212924 140540003882176 submission.py:307] 69500) loss = 0.934, grad_norm = 0.500 +I0912 05:23:36.037909 140516691797760 logging_writer.py:48] [70000] global_step=70000, grad_norm=0.499999, loss=0.947724 +I0912 05:23:36.041561 140540003882176 submission.py:307] 70000) loss = 0.948, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 05:27:52.268685 140516691797760 logging_writer.py:48] [70500] global_step=70500, grad_norm=0.499999, loss=1.0017 +I0912 05:27:52.275426 140540003882176 submission.py:307] 70500) loss = 1.002, grad_norm = 0.500 +I0912 05:35:25.296905 140516683405056 logging_writer.py:48] [71000] global_step=71000, grad_norm=0.499999, loss=0.883928 +I0912 05:35:25.300519 140540003882176 submission.py:307] 71000) loss = 0.884, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 05:39:58.544570 140516691797760 logging_writer.py:48] [71500] global_step=71500, grad_norm=0.5, loss=0.87672 +I0912 05:39:58.551035 140540003882176 submission.py:307] 71500) loss = 0.877, grad_norm = 0.500 +I0912 05:41:07.312528 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 05:41:22.595413 140540003882176 spec.py:346] Evaluating on the validation split. +I0912 05:41:42.349770 140540003882176 spec.py:363] Evaluating on the test split. +I0912 05:41:52.667872 140540003882176 submission_runner.py:516] Time since start: 53913.57s, Step: 71609, {'train/ctc_loss': 0.0706149747598593, 'train/wer': 0.028593483471163604, 'validation/ctc_loss': 0.33566981356427067, 'validation/wer': 0.08841790180080143, 'validation/num_examples': 5348, 'test/ctc_loss': 0.178314276324279, 'test/wer': 0.05390693234212825, 'test/num_examples': 2472, 'score': 52387.277347803116, 'total_duration': 53913.56525182724, 'accumulated_submission_time': 52387.277347803116, 'accumulated_eval_time': 1414.4519746303558, 'accumulated_logging_time': 6.284870862960815} +I0912 05:41:53.012490 140516691797760 logging_writer.py:48] [71609] accumulated_eval_time=1414.45, accumulated_logging_time=6.28487, accumulated_submission_time=52387.3, global_step=71609, preemption_count=0, score=52387.3, test/ctc_loss=0.178314, test/num_examples=2472, test/wer=0.0539069, total_duration=53913.6, train/ctc_loss=0.070615, train/wer=0.0285935, validation/ctc_loss=0.33567, validation/num_examples=5348, validation/wer=0.0884179 +I0912 05:48:02.629990 140516683405056 logging_writer.py:48] [72000] global_step=72000, grad_norm=0.5, loss=0.974582 +I0912 05:48:02.633672 140540003882176 submission.py:307] 72000) loss = 0.975, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 05:52:52.560035 140516691797760 logging_writer.py:48] [72500] global_step=72500, grad_norm=0.499999, loss=0.966247 +I0912 05:52:52.566781 140540003882176 submission.py:307] 72500) loss = 0.966, grad_norm = 0.500 +I0912 05:59:49.201334 140516683405056 logging_writer.py:48] [73000] global_step=73000, grad_norm=0.499999, loss=0.928643 +I0912 05:59:49.205072 140540003882176 submission.py:307] 73000) loss = 0.929, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 06:04:55.732046 140516691797760 logging_writer.py:48] [73500] global_step=73500, grad_norm=0.499999, loss=0.856052 +I0912 06:04:55.738802 140540003882176 submission.py:307] 73500) loss = 0.856, grad_norm = 0.500 +I0912 06:11:05.229962 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 06:11:20.562759 140540003882176 spec.py:346] Evaluating on the validation split. +I0912 06:11:40.301721 140540003882176 spec.py:363] Evaluating on the test split. +I0912 06:11:50.674119 140540003882176 submission_runner.py:516] Time since start: 55711.57s, Step: 73963, {'train/ctc_loss': 0.06983886421480755, 'train/wer': 0.028242610066288084, 'validation/ctc_loss': 0.33552446723678925, 'validation/wer': 0.08823444213778786, 'validation/num_examples': 5348, 'test/ctc_loss': 0.1773794629895477, 'test/wer': 0.053378831271708, 'test/num_examples': 2472, 'score': 54135.1698012352, 'total_duration': 55711.57153177261, 'accumulated_submission_time': 54135.1698012352, 'accumulated_eval_time': 1459.8960573673248, 'accumulated_logging_time': 6.639477014541626} +I0912 06:11:50.987748 140516691797760 logging_writer.py:48] [73963] accumulated_eval_time=1459.9, accumulated_logging_time=6.63948, accumulated_submission_time=54135.2, global_step=73963, preemption_count=0, score=54135.2, test/ctc_loss=0.177379, test/num_examples=2472, test/wer=0.0533788, total_duration=55711.6, train/ctc_loss=0.0698389, train/wer=0.0282426, validation/ctc_loss=0.335524, validation/num_examples=5348, validation/wer=0.0882344 +I0912 06:12:29.778324 140516683405056 logging_writer.py:48] [74000] global_step=74000, grad_norm=0.499999, loss=0.918376 +I0912 06:12:29.781562 140540003882176 submission.py:307] 74000) loss = 0.918, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 06:17:59.679616 140516691797760 logging_writer.py:48] [74500] global_step=74500, grad_norm=0.499999, loss=0.916469 +I0912 06:17:59.686257 140540003882176 submission.py:307] 74500) loss = 0.916, grad_norm = 0.500 +I0912 06:24:25.118165 140516683405056 logging_writer.py:48] [75000] global_step=75000, grad_norm=0.499999, loss=0.958806 +I0912 06:24:25.122009 140540003882176 submission.py:307] 75000) loss = 0.959, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 06:30:10.686594 140516691797760 logging_writer.py:48] [75500] global_step=75500, grad_norm=0.499999, loss=0.996507 +I0912 06:30:10.693312 140540003882176 submission.py:307] 75500) loss = 0.997, grad_norm = 0.500 +I0912 06:36:12.359785 140516683405056 logging_writer.py:48] [76000] global_step=76000, grad_norm=0.499999, loss=0.953021 +I0912 06:36:12.363497 140540003882176 submission.py:307] 76000) loss = 0.953, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 06:40:58.859456 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +I0912 06:41:13.216119 140540003882176 spec.py:346] Evaluating on the validation split. +I0912 06:41:33.139917 140540003882176 spec.py:363] Evaluating on the test split. +I0912 06:41:43.338393 140540003882176 submission_runner.py:516] Time since start: 57504.24s, Step: 76310, {'train/ctc_loss': 0.06970527802205236, 'train/wer': 0.028274998380584285, 'validation/ctc_loss': 0.3351290447778531, 'validation/wer': 0.08841790180080143, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17714690010242595, 'test/wer': 0.053521012329128835, 'test/num_examples': 2472, 'score': 55880.356978178024, 'total_duration': 57504.23585700989, 'accumulated_submission_time': 55880.356978178024, 'accumulated_eval_time': 1504.374971628189, 'accumulated_logging_time': 6.962735891342163} +I0912 06:41:43.410159 140516691797760 logging_writer.py:48] [76310] accumulated_eval_time=1504.37, accumulated_logging_time=6.96274, accumulated_submission_time=55880.4, global_step=76310, preemption_count=0, score=55880.4, test/ctc_loss=0.177147, test/num_examples=2472, test/wer=0.053521, total_duration=57504.2, train/ctc_loss=0.0697053, train/wer=0.028275, validation/ctc_loss=0.335129, validation/num_examples=5348, validation/wer=0.0884179 +I0912 06:43:08.694205 140516683405056 logging_writer.py:48] [76500] global_step=76500, grad_norm=0.499999, loss=0.950233 +I0912 06:43:08.697422 140540003882176 submission.py:307] 76500) loss = 0.950, grad_norm = 0.500 +I0912 06:49:03.204613 140516691797760 logging_writer.py:48] [77000] global_step=77000, grad_norm=0.499999, loss=0.970383 +I0912 06:49:03.208258 140540003882176 submission.py:307] 77000) loss = 0.970, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 06:55:26.513946 140516691797760 logging_writer.py:48] [77500] global_step=77500, grad_norm=0.499999, loss=0.934066 +I0912 06:55:26.520514 140540003882176 submission.py:307] 77500) loss = 0.934, grad_norm = 0.500 +I0912 07:00:50.010068 140516683405056 logging_writer.py:48] [78000] global_step=78000, grad_norm=0.499999, loss=0.885555 +I0912 07:00:50.013858 140540003882176 submission.py:307] 78000) loss = 0.886, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 07:07:35.974026 140516691797760 logging_writer.py:48] [78500] global_step=78500, grad_norm=0.467467, loss=0.932844 +I0912 07:07:35.980745 140540003882176 submission.py:307] 78500) loss = 0.933, grad_norm = 0.467 +I0912 07:10:52.957664 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 07:11:08.231353 140540003882176 spec.py:346] Evaluating on the validation split. +I0912 07:11:28.306664 140540003882176 spec.py:363] Evaluating on the test split. +I0912 07:11:38.517234 140540003882176 submission_runner.py:516] Time since start: 59299.41s, Step: 78867, {'train/ctc_loss': 0.06961636148785767, 'train/wer': 0.02820482369960918, 'validation/ctc_loss': 0.33501769271461224, 'validation/wer': 0.08834065562690098, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17730220246064263, 'test/wer': 0.05368350496618122, 'test/num_examples': 2472, 'score': 57625.574048280716, 'total_duration': 59299.41465139389, 'accumulated_submission_time': 57625.574048280716, 'accumulated_eval_time': 1549.9344420433044, 'accumulated_logging_time': 7.044204473495483} +I0912 07:11:38.797309 140516691797760 logging_writer.py:48] [78867] accumulated_eval_time=1549.93, accumulated_logging_time=7.0442, accumulated_submission_time=57625.6, global_step=78867, preemption_count=0, score=57625.6, test/ctc_loss=0.177302, test/num_examples=2472, test/wer=0.0536835, total_duration=59299.4, train/ctc_loss=0.0696164, train/wer=0.0282048, validation/ctc_loss=0.335018, validation/num_examples=5348, validation/wer=0.0883407 +I0912 07:13:29.143160 140516683405056 logging_writer.py:48] [79000] global_step=79000, grad_norm=0.5, loss=1.01004 +I0912 07:13:29.147021 140540003882176 submission.py:307] 79000) loss = 1.010, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 07:20:27.238502 140516691797760 logging_writer.py:48] [79500] global_step=79500, grad_norm=0.5, loss=0.925051 +I0912 07:20:27.245118 140540003882176 submission.py:307] 79500) loss = 0.925, grad_norm = 0.500 +I0912 07:25:22.166040 140516683405056 logging_writer.py:48] [80000] global_step=80000, grad_norm=0.499999, loss=0.921314 +I0912 07:25:22.170013 140540003882176 submission.py:307] 80000) loss = 0.921, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 07:32:32.864571 140516691797760 logging_writer.py:48] [80500] global_step=80500, grad_norm=0.499999, loss=0.920835 +I0912 07:32:32.871391 140540003882176 submission.py:307] 80500) loss = 0.921, grad_norm = 0.500 +I0912 07:37:12.991255 140516683405056 logging_writer.py:48] [81000] global_step=81000, grad_norm=0.499999, loss=0.901427 +I0912 07:37:12.995110 140540003882176 submission.py:307] 81000) loss = 0.901, grad_norm = 0.500 +I0912 07:40:46.687949 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 07:41:02.730925 140540003882176 spec.py:346] Evaluating on the validation split. +I0912 07:41:22.438824 140540003882176 spec.py:363] Evaluating on the test split. +I0912 07:41:32.691741 140540003882176 submission_runner.py:516] Time since start: 61093.59s, Step: 81215, {'train/ctc_loss': 0.06938167072185956, 'train/wer': 0.027918726923326063, 'validation/ctc_loss': 0.3353849337813316, 'validation/wer': 0.088688263409453, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17763241529695464, 'test/wer': 0.0538866207624967, 'test/num_examples': 2472, 'score': 59370.740894556046, 'total_duration': 61093.5891122818, 'accumulated_submission_time': 59370.740894556046, 'accumulated_eval_time': 1595.9380114078522, 'accumulated_logging_time': 7.333662748336792} +I0912 07:41:32.775929 140516691797760 logging_writer.py:48] [81215] accumulated_eval_time=1595.94, accumulated_logging_time=7.33366, accumulated_submission_time=59370.7, global_step=81215, preemption_count=0, score=59370.7, test/ctc_loss=0.177632, test/num_examples=2472, test/wer=0.0538866, total_duration=61093.6, train/ctc_loss=0.0693817, train/wer=0.0279187, validation/ctc_loss=0.335385, validation/num_examples=5348, validation/wer=0.0886883 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 07:45:21.356612 140516691797760 logging_writer.py:48] [81500] global_step=81500, grad_norm=0.499999, loss=0.892996 +I0912 07:45:21.363190 140540003882176 submission.py:307] 81500) loss = 0.893, grad_norm = 0.500 +I0912 07:50:04.021375 140516683405056 logging_writer.py:48] [82000] global_step=82000, grad_norm=0.499999, loss=0.934037 +I0912 07:50:04.025162 140540003882176 submission.py:307] 82000) loss = 0.934, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 07:57:35.406087 140516691797760 logging_writer.py:48] [82500] global_step=82500, grad_norm=0.499999, loss=0.954188 +I0912 07:57:35.412678 140540003882176 submission.py:307] 82500) loss = 0.954, grad_norm = 0.500 +I0912 08:01:54.163948 140516683405056 logging_writer.py:48] [83000] global_step=83000, grad_norm=0.457922, loss=0.941821 +I0912 08:01:54.167693 140540003882176 submission.py:307] 83000) loss = 0.942, grad_norm = 0.458 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 08:09:46.790315 140516691797760 logging_writer.py:48] [83500] global_step=83500, grad_norm=0.499999, loss=0.94271 +I0912 08:09:46.796746 140540003882176 submission.py:307] 83500) loss = 0.943, grad_norm = 0.500 +I0912 08:10:40.969510 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 08:10:54.920265 140540003882176 spec.py:346] Evaluating on the validation split. +I0912 08:11:14.434984 140540003882176 spec.py:363] Evaluating on the test split. +I0912 08:11:24.951329 140540003882176 submission_runner.py:516] Time since start: 62885.85s, Step: 83625, {'train/ctc_loss': 0.06910094946475681, 'train/wer': 0.027988901604301168, 'validation/ctc_loss': 0.3360513783325171, 'validation/wer': 0.08826340945300053, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17802338054068004, 'test/wer': 0.054495968151443135, 'test/num_examples': 2472, 'score': 61116.29442381859, 'total_duration': 62885.848786354065, 'accumulated_submission_time': 61116.29442381859, 'accumulated_eval_time': 1639.9197208881378, 'accumulated_logging_time': 7.427438497543335} +I0912 08:11:25.303332 140516691797760 logging_writer.py:48] [83625] accumulated_eval_time=1639.92, accumulated_logging_time=7.42744, accumulated_submission_time=61116.3, global_step=83625, preemption_count=0, score=61116.3, test/ctc_loss=0.178023, test/num_examples=2472, test/wer=0.054496, total_duration=62885.8, train/ctc_loss=0.0691009, train/wer=0.0279889, validation/ctc_loss=0.336051, validation/num_examples=5348, validation/wer=0.0882634 +I0912 08:14:57.906311 140516683405056 logging_writer.py:48] [84000] global_step=84000, grad_norm=0.5, loss=0.99534 +I0912 08:14:57.910110 140540003882176 submission.py:307] 84000) loss = 0.995, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 08:22:58.592951 140516691797760 logging_writer.py:48] [84500] global_step=84500, grad_norm=0.499999, loss=0.951043 +I0912 08:22:58.599694 140540003882176 submission.py:307] 84500) loss = 0.951, grad_norm = 0.500 +I0912 08:27:00.447012 140516683405056 logging_writer.py:48] [85000] global_step=85000, grad_norm=0.499999, loss=0.935183 +I0912 08:27:00.456072 140540003882176 submission.py:307] 85000) loss = 0.935, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 08:35:07.200563 140516691797760 logging_writer.py:48] [85500] global_step=85500, grad_norm=0.499999, loss=0.935491 +I0912 08:35:07.207512 140540003882176 submission.py:307] 85500) loss = 0.935, grad_norm = 0.500 +I0912 08:39:01.765958 140516683405056 logging_writer.py:48] [86000] global_step=86000, grad_norm=0.499999, loss=0.915837 +I0912 08:39:01.769698 140540003882176 submission.py:307] 86000) loss = 0.916, grad_norm = 0.500 +I0912 08:40:34.760816 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 08:40:50.699194 140540003882176 spec.py:346] Evaluating on the validation split. +I0912 08:41:10.428956 140540003882176 spec.py:363] Evaluating on the test split. +I0912 08:41:20.657049 140540003882176 submission_runner.py:516] Time since start: 64681.55s, Step: 86130, {'train/ctc_loss': 0.06903047683572347, 'train/wer': 0.028102260704337873, 'validation/ctc_loss': 0.3365171929547795, 'validation/wer': 0.08863998455076522, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17861006153622813, 'test/wer': 0.054495968151443135, 'test/num_examples': 2472, 'score': 62861.324580192566, 'total_duration': 64681.55446791649, 'accumulated_submission_time': 62861.324580192566, 'accumulated_eval_time': 1685.8157725334167, 'accumulated_logging_time': 7.788813591003418} +I0912 08:41:21.022328 140516691797760 logging_writer.py:48] [86130] accumulated_eval_time=1685.82, accumulated_logging_time=7.78881, accumulated_submission_time=62861.3, global_step=86130, preemption_count=0, score=62861.3, test/ctc_loss=0.17861, test/num_examples=2472, test/wer=0.054496, total_duration=64681.6, train/ctc_loss=0.0690305, train/wer=0.0281023, validation/ctc_loss=0.336517, validation/num_examples=5348, validation/wer=0.08864 +I0912 08:47:43.929072 140516683405056 logging_writer.py:48] [86500] global_step=86500, grad_norm=0.5, loss=0.948667 +I0912 08:47:43.932583 140540003882176 submission.py:307] 86500) loss = 0.949, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 08:51:51.743380 140516691797760 logging_writer.py:48] [87000] global_step=87000, grad_norm=0.499999, loss=0.953225 +I0912 08:51:51.750434 140540003882176 submission.py:307] 87000) loss = 0.953, grad_norm = 0.500 +I0912 08:59:30.895189 140516683405056 logging_writer.py:48] [87500] global_step=87500, grad_norm=0.499999, loss=0.928146 +I0912 08:59:30.898781 140540003882176 submission.py:307] 87500) loss = 0.928, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 09:03:50.377487 140516691797760 logging_writer.py:48] [88000] global_step=88000, grad_norm=0.5, loss=1.01514 +I0912 09:03:50.384280 140540003882176 submission.py:307] 88000) loss = 1.015, grad_norm = 0.500 +I0912 09:10:28.653791 140540003882176 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 09:10:43.642092 140540003882176 spec.py:346] Evaluating on the validation split. +I0912 09:11:03.139639 140540003882176 spec.py:363] Evaluating on the test split. +I0912 09:11:13.339941 140540003882176 submission_runner.py:516] Time since start: 66474.24s, Step: 88461, {'train/ctc_loss': 0.06960582494461544, 'train/wer': 0.028161639280547577, 'validation/ctc_loss': 0.3386531168374523, 'validation/wer': 0.0890841500506928, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17728859271162656, 'test/wer': 0.054089736558812176, 'test/num_examples': 2472, 'score': 64606.37130188942, 'total_duration': 66474.23734116554, 'accumulated_submission_time': 64606.37130188942, 'accumulated_eval_time': 1730.5017790794373, 'accumulated_logging_time': 8.163705825805664} +I0912 09:11:13.422515 140516691797760 logging_writer.py:48] [88461] accumulated_eval_time=1730.5, accumulated_logging_time=8.16371, accumulated_submission_time=64606.4, global_step=88461, preemption_count=0, score=64606.4, test/ctc_loss=0.177289, test/num_examples=2472, test/wer=0.0540897, total_duration=66474.2, train/ctc_loss=0.0696058, train/wer=0.0281616, validation/ctc_loss=0.338653, validation/num_examples=5348, validation/wer=0.0890842 +I0912 09:11:54.760686 140516683405056 logging_writer.py:48] [88500] global_step=88500, grad_norm=0.499999, loss=0.951315 +I0912 09:11:54.764120 140540003882176 submission.py:307] 88500) loss = 0.951, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 09:16:40.919152 140516691797760 logging_writer.py:48] [89000] global_step=89000, grad_norm=0.499999, loss=0.948031 +I0912 09:16:40.925826 140540003882176 submission.py:307] 89000) loss = 0.948, grad_norm = 0.500 +I0912 09:23:52.087515 140516683405056 logging_writer.py:48] [89500] global_step=89500, grad_norm=0.5, loss=1.01579 +I0912 09:23:52.091342 140540003882176 submission.py:307] 89500) loss = 1.016, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 09:28:47.540001 140516691797760 logging_writer.py:48] [90000] global_step=90000, grad_norm=0.499999, loss=0.912644 +I0912 09:28:47.549567 140540003882176 submission.py:307] 90000) loss = 0.913, grad_norm = 0.500 +I0912 09:35:37.651000 140516683405056 logging_writer.py:48] [90500] global_step=90500, grad_norm=0.499999, loss=0.923532 +I0912 09:35:37.654932 140540003882176 submission.py:307] 90500) loss = 0.924, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0912 09:40:20.492769 140516691797760 logging_writer.py:48] [90928] global_step=90928, preemption_count=0, score=66351.7 +I0912 09:40:21.790194 140540003882176 submission_runner.py:857] Final librispeech_conformer score: 66351.72258377075 +[W912 09:40:33.490403054 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator()) +[W912 09:40:33.490411704 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator()) +[W912 09:40:33.490435731 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator()) +[W912 09:40:33.490451678 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator()) diff --git a/logs/self_tuning/ademamix_golden/study_1/librispeech_conformer_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_1/librispeech_conformer_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..5b3d2be6 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/librispeech_conformer_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,39 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/ctc_loss,test/num_examples,test/wer,total_duration,train/ctc_loss,train/wer,validation/ctc_loss,validation/num_examples,validation/wer +66.33343935012817,0.0,10.965123414993286,1,0,10.965123414993286,31.58821859474805,2472,1.2729876302480043,78.10448575019836,32.56430697422715,1.068215187960184,31.449624522804903,5348,1.2736831941292908 +110.83208703994752,0.0343098640441894,1757.0212490558624,1616,0,1757.0212490558624,3.992824133815085,2472,0.7336542562915118,1870.887146472931,4.018316224717581,0.765940448686114,4.212096738748242,5348,0.7391976053686091 +156.67479920387268,0.0733029842376709,3503.4649250507355,3833,0,3503.4649250507355,0.7281453642170462,2472,0.2421952755265777,3665.88813328743,0.7888560762362172,0.2638028199425647,0.9988812796677216,5348,0.3018104572007917 +201.2211222648621,0.1294331550598144,5250.175142049789,6123,0,5250.175142049789,0.4199425920623911,2472,0.1415920216115207,5459.857209920883,0.4452975969018467,0.1541953663118347,0.668249043255224,5348,0.2019601216627239 +245.47307872772217,0.2418584823608398,6995.479728221893,8655,0,6995.479728221893,0.3605698855571327,2472,0.1206710945910263,7252.397528409958,0.3602500813857133,0.1267030855267419,0.5876604156463482,5348,0.1749336165693043 +290.6490111351013,0.3359074592590332,8741.733561992645,11059,0,8741.733561992645,0.3231082700900058,2472,0.1078951110027826,9046.708485126495,0.3148550858672123,0.112921857793708,0.53931664759456,5348,0.1607299763433592 +334.5619010925293,0.4564213752746582,10488.446098566055,13355,0,10488.446098566055,0.3071018190689722,2472,0.1025734771393171,10841.1125998497,0.2846758628999526,0.1035454408049575,0.5182867067416366,5348,0.1547433978660744 +379.58304166793823,0.8120191097259521,12233.473339557648,15907,0,12233.473339557648,0.2902662318939931,2472,0.0983486685759551,12634.514830112455,0.257056379160184,0.0942445965495649,0.4925074777570574,5348,0.1475401921498575 +424.8829026222229,0.9918942451477052,13980.496501922607,18300,0,13980.496501922607,0.2783933640880057,2472,0.0924786220624378,14430.19681930542,0.2425476053575052,0.0895698831861464,0.4747996055291842,5348,0.142287452324627 +469.6380949020386,1.3005595207214355,15726.650465726852,20592,0,15726.650465726852,0.2681596897380476,2472,0.0874616618934454,16225.020930051804,0.2238051361783805,0.0831623950078811,0.4520509676543349,5348,0.1334234538695505 +514.7942316532135,1.469043493270874,17472.34481048584,23169,0,17472.34481048584,0.2625980658026324,2472,0.0869741839822883,18019.78716826439,0.2137061732648988,0.0779640705633407,0.4544447426913803,5348,0.1325544344131704 +560.9092576503754,1.828929662704468,19217.717973709103,25545,0,19217.717973709103,0.2519238136331376,2472,0.0829728027948733,19814.8000998497,0.2012373931416153,0.075707684667372,0.4330134438730661,5348,0.1277844831748177 +604.724237203598,2.0684447288513184,20962.9728038311,27875,0,20962.9728038311,0.2433782018194722,2472,0.078402697377775,21607.65250182152,0.1893225379236285,0.0712812817135577,0.4232125982645168,5348,0.1242601264906097 +650.1285529136658,2.2313308715820312,22708.710742235184,30432,0,22708.710742235184,0.2367694337134653,2472,0.0762090467775678,23403.356159448624,0.1802439880267875,0.0678589165029257,0.4170667001802039,5348,0.1208323275237773 +695.7054653167725,2.575826644897461,24454.32293319702,32790,0,24454.32293319702,0.2304708922753081,2472,0.0751731562163589,25198.224741458893,0.1686431284034025,0.0642908038779608,0.4079771833245429,5348,0.118881861632791 +739.260491847992,2.649347305297852,26199.73352622986,35167,0,26199.73352622986,0.227936026114588,2472,0.0728982592976255,26990.98868870735,0.1633243077986538,0.0618454861485975,0.401488988016752,5348,0.115618210785497 +785.2804026603699,2.9802584648132324,27945.150611400604,37696,0,27945.150611400604,0.2228330255177753,2472,0.0707046086974184,28787.031717062,0.15306217709954,0.0585202858808541,0.398219505035664,5348,0.1136580891227731 +829.5948066711426,3.313499689102173,29690.69269633293,40029,0,29690.69269633293,0.2162197712352409,2472,0.0689781244287368,30579.836644887924,0.1467357326616722,0.0564528318182799,0.3883773561507936,5348,0.1106744556558683 +874.6922862529755,3.3926544189453125,31436.16641187668,42457,0,31436.16641187668,0.2129698387799213,2472,0.0666829159303719,32374.587361335754,0.1373192074511263,0.0533489516982272,0.3837925943716094,5348,0.1090426302322213 +920.2076592445374,3.678757905960083,33182.07935786247,44951,0,33182.07935786247,0.2074357619846441,2472,0.0668047854081611,34170.63980221748,0.1291461132508624,0.0502072852114956,0.3754650052350186,5348,0.1060300294501038 +964.7020807266236,4.014244794845581,34929.89146471024,47271,0,34929.89146471024,0.2035757851312988,2472,0.0639002295208498,35965.88739991188,0.1227502927771764,0.0487552091205493,0.3674541563065099,5348,0.1036353980591898 +1009.7656865119934,4.100290775299072,36675.30523777008,49757,0,36675.30523777008,0.1986226051882379,2472,0.0615643978632218,37760.6712975502,0.1135258299228843,0.0453112517003865,0.3629400810132861,5348,0.1012890455269637 +1055.598946094513,4.455331563949585,38422.19172549248,52211,0,38422.19172549248,0.195299028868798,2472,0.0608738041557491,39558.31389641762,0.1048411551697896,0.0417593332325696,0.354769562055769,5348,0.0994930719837783 +1100.216621875763,4.762305021286011,40169.08051586151,54520,0,40169.08051586151,0.1944409593965739,2472,0.0598582251741717,41352.719628334045,0.0985952905152878,0.0396055103318722,0.3567766707322433,5348,0.0972239656254526 +1144.8478198051453,4.847512245178223,41914.625618219376,57045,0,41914.625618219376,0.1904134062076585,2472,0.0587613998740682,43147.09202218056,0.0929072879320841,0.0373113380692246,0.3505628822269941,5348,0.0952059093323033 +1190.7182912826538,5.198246240615845,43659.64862132072,59464,0,43659.64862132072,0.1856567611216852,2472,0.0571770966628074,44942.64661693573,0.0865763316076743,0.0347904476065035,0.3435994049188768,5348,0.0935258050499686 +1234.8561778068542,5.529304265975952,45405.07888031006,61769,0,45405.07888031006,0.1818494716392993,2472,0.0564458797960717,46735.14002013207,0.0820577201747784,0.032782372120139,0.3425308420688291,5348,0.0926760971370636 +1280.0275931358335,5.600770711898804,47150.370564222336,64336,0,47150.370564222336,0.1816025429059939,2472,0.0547803302662848,48530.06073570252,0.0778430698479672,0.031557014229266,0.3417285430951125,5348,0.0907835658765026 +1325.4059286117554,5.914076328277588,48896.60098361969,66723,0,48896.60098361969,0.1792037742354345,2472,0.0548006418459163,50326.60763835907,0.074476178560255,0.0303208602336277,0.3377734845916214,5348,0.0900304156809733 +1369.0967593193054,6.2146546840667725,50642.02622580528,69036,0,50642.02622580528,0.1782793950416155,2472,0.0540288018199175,52118.62122583389,0.0720835310724311,0.029090104290372,0.3336585107902602,5348,0.0886399845507652 +1414.4519746303558,6.284870862960815,52387.27734780312,71609,0,52387.27734780312,0.178314276324279,2472,0.0539069323421282,53913.56525182724,0.0706149747598593,0.0285934834711636,0.3356698135642706,5348,0.0884179018008014 +1459.8960573673248,6.639477014541626,54135.1698012352,73963,0,54135.1698012352,0.1773794629895477,2472,0.053378831271708,55711.57153177261,0.0698388642148075,0.028242610066288,0.3355244672367892,5348,0.0882344421377878 +1504.374971628189,6.962735891342163,55880.35697817802,76310,0,55880.35697817802,0.1771469001024259,2472,0.0535210123291288,57504.23585700989,0.0697052780220523,0.0282749983805842,0.3351290447778531,5348,0.0884179018008014 +1549.9344420433044,7.044204473495483,57625.57404828072,78867,0,57625.57404828072,0.1773022024606426,2472,0.0536835049661812,59299.41465139389,0.0696163614878576,0.0282048236996091,0.3350176927146122,5348,0.0883406556269009 +1595.9380114078522,7.333662748336792,59370.740894556046,81215,0,59370.740894556046,0.1776324152969546,2472,0.0538866207624967,61093.5891122818,0.0693816707218595,0.027918726923326,0.3353849337813316,5348,0.088688263409453 +1639.9197208881378,7.427438497543335,61116.29442381859,83625,0,61116.29442381859,0.17802338054068,2472,0.0544959681514431,62885.84878635407,0.0691009494647568,0.0279889016043011,0.3360513783325171,5348,0.0882634094530005 +1685.8157725334167,7.788813591003418,62861.324580192566,86130,0,62861.324580192566,0.1786100615362281,2472,0.0544959681514431,64681.55446791649,0.0690304768357234,0.0281022607043378,0.3365171929547795,5348,0.0886399845507652 +1730.5017790794373,8.163705825805664,64606.37130188942,88461,0,64606.37130188942,0.17728859271162656,2472,0.054089736558812176,66474.23734116554,0.06960582494461544,0.028161639280547577,0.3386531168374523,5348,0.0890841500506928 diff --git a/logs/self_tuning/ademamix_golden/study_1/librispeech_conformer_pytorch/trial_1/events.out.tfevents.1789137799.bc2a5acb9a19.38.0.v2 b/logs/self_tuning/ademamix_golden/study_1/librispeech_conformer_pytorch/trial_1/events.out.tfevents.1789137799.bc2a5acb9a19.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..c047602629eba8bcb4fab605c646089e4d49232d GIT binary patch literal 100105 zcmbT9c|Z+c8^=Sm2uVn`h$JL?mhAgZk+r0SENP{Pq9heU2uT!5A+m%bgd!x$7E)A* zvX+GR*W{ghzH`ppihqCa^Lghp=iKM{-nlc=K$(hvPtpeGDagnqypoN0Iz+wRe0wu% z8{HAhEo~h8b)VW!v8|(}jl(i~t0l`;=$bFHx72lVv|gfXS^nQ0Ojp=jJ6hV0Yw_hu z=fAYC$5Tcq%!>7RT~5p39OWXe=%KyiJ$cT?#tp67g>ayBc!Kc=wZ{^3YTtlam zVn@aLj^*d1tGT1ODe3ZbNKJGZ(sg=2IFwlpcn)V%!C7Hx|Nlc7dC_|i97;%UJB4ETp)sqj! zgX-69iP@092PG46HBzBPN1oGDvAbeZf@9;f)YQ`1Y^m)MOMcO1G>(jb1Cwps#;(s_ zU|#;MnF{yxc%Y7=wqiX;O9w~ld`F!wL%)RKUHrTM#k6ZSy+EO*K96aysHrH=VFiCeO4 z!2)ZW1wbsXg$=cGo_Bh$zr>8|b8{8WEar*gZNMEj!<@*6WBUDtsO)%`t#HMjhjdZw zq^N9bZ)v&Iw*0eaYQD_L#_|7)KpoD|ITP0C{v}<%ZCtA0T<`h+8w%AD8>Y$P>1nZO zT?Dj|(M|Yjc2g`9Hbsp^ZMwqw$Zp9as(e1o%v`g;V*iXf-?_F}lWy!^-q%5k6^nlB zf`yzJU(L4oBeYPt@N6tt99zgE#4Th7!`67UDIpvz7C&rTVNr{=@N*K3Cw<@Ev0xFD z$yZm=V)x1Bv%q4|YaStPF&HgMt}BfLi>RrCDlBAa3zL?$aV_>7?$8JF@z8woh}eu&6^@m|qoGblATBF<6++<*V5i)LO{z@4pKyK3VVx zaf<}Bcp92I1T4HC1yoqb(H3bzf~fOG=XE?-D7@sW*%s7VY+uo$A6V?o;}PN(S!l6Z z^LYwb?4NzI!a|<5*w9YUX&uuK?gWco=6p5Vf?A7RO^$W}iwBE%gt&za4*xrSW440D zyKPS@EEH&qh_iyVIOO%`5LlS*<*V5i|BWUbeINd4%7Df2P#z&}p^O%KM+{WJLgmt! zVs2h2(iTUy3oIU%489B&*RS){RkXNYR(AndWZ&Tt;uacc@n>)5VXz3@;!|N!m$uOE zPzyV4&9qmuz+z$`U(L3lc2wwwvmL;U$D6C@x8*L0d1k}Citq>;d=M+U}59SSFjTiBvSw?)0zf<@MMl^@)WQl>2ix%|QE;Y+S<1dFA=`RXcK^r)FR6)ZZ| zY4raC5Vvs0<527Jr#D!9DhjE9G^8y?CzN7!OwP`1u<$G4tJxORycn=ly#y@E{_qHK z3vWD*(ynD6!9pcerIcF>723jWWieKF>zLgiEUp~otE*_Swbq{gU=ewWM~GYa<8ka7 zliUd`UU!>aVbO@T_&T)^tMgiQs0S8#efes(1^v0u48U(m--qxBaf?7aj+t)zY`|i| z`c)Mcs}m z4@N%2>gTN+yMjezDqqdEpgtG1XRL|^3%|!aLfj%8k7Hf!L&sn(YMu4|$$c)G&=#kA zJ;3VT%IcWR-7fLfRkR5DRpt#A^=|M8af@g?j?>=qqru{}*YOIArnJSj-~_C$F-jYA z)R)bCHQR#vTdY=BSJ$i!yFUHKQ%+j|fAH!?{{N!6M}WU(L3lJ{LZB zp5S*q%}gF4Zjpq?(f0Y+9N1CI!)1SQ7V5NxXWIa*&hwb04;JH2^3_$e(CVKQ4{I^z zJdY5!NX6qw(MoI%7OoxJR9G~pEf$A)Vs*{=k$4THQxCqHZ9#o5>bsvZ0E_+oc!apc zV?2&eign__;@A;`3X2xBMe3P_Sp7MtV^>%UixYfx6)ifr&1ni27tiqsaf>WGj+-gj zi@_q(GOfa*C2f(tW-M0c?DxRyv@4hL)ocsubJ6C|*F9jd)QLxkTjb+$e6t+u3Kj|N z%zks9i&nHnQXf66F83)| z&9#otKipQG1;d@4bDP~wrH;Q7A>agCF6D4h1>XQwgvUMkeBOJ7c3n2 z@Cb1WHMCeiC&Csi43zs;SahH*&N>U$;??W}3t%mtsqxiSv}nJ4&m6GG*5DE17A??X z^zupH!9qS@T7`u+ZDBnl7uTZ4uGOEw;#3e{T}2D!339Gru_}y5h+Al&#W~B)XTjpk z#P|w}jRG~p5A7VXf&RI6bySm@fVuE~8a zbZ86hWWg^{4@O%|0gDzcd^OvGx|ONq?fy`(xa`gWVZ;%8nrHL#d>a!-XtC)#51 zqL;W9T73+n!J_9yzM5@OD*UB*VD_6-uqe68Bg8E_qs8c+$78_4toOMJi_Ww~vWmcB zTYSU(L3lcGSdn9pk`4dnAt#x6nfi#RJ#6gT=S;ZEA5lstawA5%dDr;+TK1 zGg#Oe@zqtd_`L1C1y~r);Su5%J<(#!a<|rCVQ7+DVWCS~wEmij7J)K@<-sD$lCNf4 zP@fB*sfSm9#hIl%Lfk?hE#%EUTm_4zxiw|EwdhJ)*cLuRi{rz;Wr9Vck9;-Tg4$8L zhqaUii#p$Vgt)~3v{2Zw9Djwm z;ugcvqS3TX8DR0p#h^B~qq@@;oBBS+wb&dKZ3GsI>-g#_TKqn}rY%@B@!=8T7NgPP z;4E9rv^q6sRao?(EnI76phde(jX_`$tjJfhElPxI;odyJ7A!ib@Cb2>@o2GS|DpY` z7DrbvudwJzTXZ)`M+-;S4VcVr*7MbD3u;I8RiCjBEb49I5#kn;(L%RL`v+hlV_Q;T z(Tlb)ef|(FW`BHA3~Ld$g0E& z6x@D*7Rk#8J_3tt=6rP(Ey8uI+k=JGVjdxGF#|2eWv07<#jK126&8JHi_dG)(4vm( zeJilI^@6WvTYMLOE|O!rSb@d+Tpl5AF$XOUZ&r8<7WrLnS6JxN7Bar~(LzmC4}VL_ z>BCpEEvV0h&1&yQU=cisM~GX@LyM_5{EvdgO7*#N+>Yu?Tbv9}MGLdG-y6eP#J1(D z*%s6_5Mv{SL159S6ORzLus{nd*Lz#QV*9ep3X6WUMUd4!w74>9Ln&B1S;1GcEsBLZ zDthBC{KmS&okxgUEJTY0f71f6=vrD*VbPzqP{_QC7Uw5#ISm$3HGcfRnr%TP5V=Xl z=V2`j6?lZW#S*kI9yIA4SZpZmD9`Pv0kp-~83KzM{q2IlLbs;Q|Et*+RHwbaG29U> zA{BUqxP>iR6wYkX1}v8UNv${z1KPsE>JF|&+q;peU@=ne{{O4l7C(eL>g!9F!C=w1 z0gn*3a72qb3JJoi%{>&jwHQcS%=Ab>i{e}HU%}#iFTR>>@l|N?NiP?7)P4gVA#UM} z7V{VT`@vewI&WNIF^IN^TzeZW9@NlD28%}td^OvG>QQNFduqX2#3u0waSKkbw(H}VK^3wN{_XYwQwEZjW$ zC~`ZB@kpA&U4ezVv04eN#du%7nr%USF227`!{1n2Zs!r=7VFT$C!r*Pi!9}Y3X5U1MSOY^ zS|~Wb!rA>&jjv`~P@jtj@!5E1(R~dbA#UM?7V~0NaCU3E7FSpdr!Dr#2-YI3b+0?H z7WX{(YPJQnqjWM4#8Ux)*HAML)wcJV8$ zMQi|%5V!D0iw5@=i~@`4+Ak_B#?TgB76>frepzV`77x1e)ocrDNA0kikIyfb^x+ZW z7TeH5b$szOu;?gfRFB(HV`&RFSAm7bWOuw1wW%^+&9?X=oEMAV+{5cp=bQ2faf<-7 z7&$X)BUtS6wXLuiM_bgoa~*fofV{u~uokM@`D(TWH80vuTQv-JRJ*-ALfm2}TI{M{ zAMeT0T##8|F`l;Q;~}t^k$rv*ShQNkSF{_4OQ4?s3PD%ocxbTYu!D2`-UtL9u^blQZu&_VOBg8HCpv85Q$92Hs*Yl?p787ZU z56uJ?-QyqSfyMqed^OvGntJsY8XJK{$_E}HZV`kQ+b2B0Pj{;&#T6ElXbZm@0t@RO z;htcT?Z8*FEvQa==2IwM;cVc_Bg8F&(W29tbC+Q)8aMCVfZI`%X^U}_67kdRmA~Z& zSR}XQtJxOR)Jrbg@degmst%73w+KOt$cdgFU||%suEJspZLw+ZHMIDw<(2>zPh$9L zwgr{U>mI9f6D<5L@(6K@gJ^MmTqHh4Quw-^GPf2}X^Vry1QxxHPVa#hANXpvMS*Z$ z^j2H<6f9PK=Mmx-VQ4XDlEyYzi~CMPDl7~c7S{z9n`Xbq@7>c^^VMt%YDZb;K70Tc zi`Mf9af`!f;bO9N5m;FL@T#ydqAiY06IfW?8-aH}TGrU`|7x}c_5EVf>rXmh(N2L! zh+BlCh3<#Nc!&17)=w*rV;XJoy}?!dT(oKa6Q3f{?!Z^GEvO##`=KB1sBgMFLfqmg zTJ*}S*%sEq%g(JK_qmu(Tj>t9Esmqbw#YwWVBt66 zX~l8Ope+XL3D%;veVw(i7BbWLYPJQnqc%+FZx0r7b9scgMKoG$)@alhEP9U7QsH*g zOxmJD2Z2R|_U!p!5jKUdW?N7RtQXj z8u8U^3+i(bH^e>?EVNqi2yu&8wAh|B%o{B3X4zF(%%v@YTm^5i$T?SL2^Piqd^Ovm zNVuaW8;;Edi*rRhLfqm4S`=^9#`l4=kT+N5J{KmmMU&0~i&4j~I)Q~>1HPJVL8ZZ- zcTPV8i-@K?Lfj$_Ewpq-B*9u-I_OqmVM<$kJ}0nfpPGrk>**fltJxOR=OSXe{7SG0 zKE)%%EiR#j$;?N%qsHuuuCSO#TeLqYut>;Wn+t34HH@#WqQ%T-8au$E{V^UPZjpc% zCtd~KgSGH!d#A#}jJEJ~yNt7YmG{93V4>HEuV!0NvpcKXCSS1l+MP#;TUTSIEIuk5L#_$Mni$t_AoZWLhSQrm` zTVY{NTU@#+u&}E83a>|f7|U0)EvR3L7Y4ap0gHG;9wBaV11&=8c#j2(;JdY(aBE>f zTl7m3Sp59)bS11s=M27@Z9(lQ+X;@?qi((65#km}Xd(0JuqV7@Praw|e$L`6<7Hks zuIKQB@n_2`>z@BTF;)K0q({>^3d0N}{qBTYlMU$E>b!S zUP|`Ia>z~)S(~Cbp-%aOB>9ZDFMUxmL6K)8A}v5<>LW=xTjYaq^esN&z7Ryzvm^|0 zk(I+i#45eE2Z;3lQ!juM$z;4PXIVcz6gkzt_G1gtt1Rl z5%-HIayH_2Sl_?v>MPghlS0YjwP!eyR3&LI0ae=Rj!H=%w7-GU&gBvir$oI4G)i?i zzK$tmt)kn&zX!v;p7r}`38S*tfVw4n;Smj-^)Lw3Jl0ESQr-hve}DlV){~T<4)ET_ zW}79Te|iz<&EgH`;H{UYJ0wK$H-XytxS?V|uUl%+pl$|9ILudpCZ_gBxnZA*#|`*< z&XdlCPF467CP7s5U7*e$x*zravUYq+HC51xlCbEPfgTTehMRGC_qrjlVhdv=^-|sj zYLWRCLC+pcIX?35f!Oyte@Nl%c?qbhuLGqHug%O&&V&C_wVQt3p>QlgLaqFLpmIl_ zV_Q$S{Bg~&zr2&{+lMIlBuRKxdLgL0w~_($rP^*&L02Uw>OiABV4^y8|6D;$9J|W zcmU^Q;0g&t{G62HIf+gkRR=_#ZaJ`m6G>#e@Wx?bJrwyfVAvWEX|PvP&K7wq6giV+ zHULC&4oMiIB7~HKBjyJEVf{5xYNqH4Tn)wuW z1d&0;0sYeq zPT%1D5N7a!ixQ&v431mS4;9y#&iD;8c1vR{*7FoP$4lJKfDgR@%i#2FlZK*kuF!lNG&q-ay{Uxuge z#{!2LFoW;P4C9+Z^=5Dz#YmjNkN4#tEJ0K= zOq|~Meiwv^EA|o=9VQku48zT+Zgxc$RxD8$ZL_)WS-jtJwYbcUb;k z&Tf?1!baItE^;EA6YVt;hUhubxQ*xJee1C`P$YP65_cqz8PDr_X5T`Q4xKHUfk;@m zq?|31BNWNI(&Ge({E3z@L`4#wqR95GbLW9bnbG?;b<0Or3gZFG)Lkhk()^mqX%I1< zCn;x(P%EM_;aN=(>99n?5EYsC4n>x>xZ(>UD=tL2a3TqeC*srF|yi@}jFBPMELYU|iE@4y_Cf?aL!IP}8yY)O!D?TBiNeL4vZF=Bg>HPE#hA`oN zUIO~3FmdSan6(fle6L7|;$b28%RyKL5)W{MCdVp1iD zYKDo0vulPynDBlqVbNhCW3QmkT{?TeCajoNmZV-vn9vQ)#*b&CPYTHpCgk%ZpsI!m z>hL1k$2mfnxL7EmRvso=xHw^$FgY4L55h!~VhOKGVIm=71#Z=gvh&tJQ)nxb%YU3j zo5BD&Yzk(@lV(Dg=qM+lR4+_CUG)Sns-HeSsGm~#=n7{%(-v2789&2IO1CV9t};DV zQqFFa)Yr#8YMnITob~Nik+1VmeI15EU8R0!7}c&ddRk+*Ugc>y>XUPsVNj=jsSV@{Cr`0Flc& zl5)1l3t_ywqcvwOh&1UXVTg(-w?mN;a(eC{^0H)3Feg&NxV!wTQao<0GZxufK%`}@ zxqLZWBwr}vclT^t5ZS6EVTg)kbwZJngKdqVe=qL1>KrE$GhJGki0*j++t{k)9x0&I zr-uZ@DN$i!`>;G5Y!hz3`vPI&OMeNYvM>>@tcWMsa(DDSP`fr#L)70>Z@3xe^XDOtdV?#B*Ml;PeK< z#BoarqMBhME%ZSKgo&@V5*8gM3U|umW_)?+0>(cL7fHR8Fp)m19)fHZ^u}8aF1t%W zRSgr=;T5QkZ-;>==5GFc=NGR0{6I1nzCqScYW?R~*etE|)U_5T{GCCGN!+Xl+{f4g6 zZk42*-6*MVT_-1f@`ZEKV}pbtdQPHy;5ku#m6Zh|?cX<^%ZX$$?hij2bPPo{Y}wum zM2>%zl(R*sAGV|S*I5Z7&&nhWQ4#n4DB@Mew?2sYXU@OPi9BT7(7V=Iup$;}Uy4EG zMZTn*E%Hh@x&|sdtOX*|zepIOB9n)qh=YIVH4u5QsIGE@@~xF*N#92*_bv`sWMS>t zIv_I6QBuwpp?-LJtf& z1WLD>OF*0wHG>Z}c!=|0kbjexFoQ!iC5*~uugQt#^aF{c=u1Xr7^D&My z$HNR>F;Rl3<_z{Zr+5KoaR2EN7CnQLUuoiI^vs{H4=c89uB2Yd40f>Sh#-TeqZQ!` zz1u9<|N5EYrX2t_`b>?(j2dDvm?Z%)L|g%+`C7KI`PLl+DM zk^JtGa<&Nd8S!tKqYEP01`>v-h@L%)tj*K4g#LYho4tO+@~su>DlJUB3G%=;w(5vR z1}NTNkxn?cB z$q*(crb-ai3=_)->=*-KV)|nViw+ZS+7H6b80#566ILuQOHwZ-Of*7VPkPg?(+ni z!VDP){^Km#6#Sg=^vzuTa5RL81UU(%dSRmAP3|&il#dR)9L4n)_tkWx9IP%FUCqo^ z7(!Rkjg*wL8zptiebZ1=S2!oLPfHl0=R{^Do)dj-%>l5thm4+eg%kPeUV}C{W+3=R z|1rrLADs^~B>oqmu}&*pqMq$S-EW}YrPF3`$?HQvL|sxn(Pewsg0tXq{d&5SO8EfG z^k%q>(}+Nq+95M;gUj=KL_OPuN=G5XTGs@Z*N+K^s7tIjx-|UMIT~E{H8eWOxup6r zTsD6YxF{#WSuLjrE~#1sMARk34_yj7cW>YC?^adW@h)2-ai!G{ z&ZTT~4Qa{3%0zHyef;`>1z=U*iy(1URJ2%Rdk)9krfX&s2mL*qDVEF46iPM`kjk<} zq1qQ5ym_tG`~o}M9RyBFxVRJ35J^RQM`pqhcpFTR{wZDf*u=MmF?#n9ffSDy+c$2; zV=L~v@hWuR&Bq83GhZyTjK>py%H#GTgTE(!-k47X3g^xcjA{mq@7-Nmfb@P869ii_aFWjIBcfw9)+{d2w;^`MpD6Q?3wv> zBBwx$nDd%oinfT7&3FnoUX7Rt(R1i~0#m)5VYJ0a8(OEz{@kvO%Ev&WKcjViP7;hX z+2^^+&}6QJ6ZP!YN&Q$n&F|w^I4?U-5)jez;_Z*;#eAHdKezOX$JjHqg|K2dZ4)T#Ej4sc7ooZ09bPxKTPC3&wZw%^E|HI4de@C`~zv6YqPQirY^D&9 z%A!VJhdex{PzU`}V3#|Sz)6W3+9C2tvRmpk9->A&bAt3wQDf-!l&26i9xNh|;!&gD z_>FjMoBL|vZD57=1c(_m)LX~ki8oN#whp3(>MDX!&8X2xbKG8t8sF9uFgj}Nx!{Dm zGDN9s0j!&rF9DDeH5z-aMv~9-hUySCWVaEdszwd!01sx`O~EE&Al(6 zbq34tddsyN?=VK|G<|sjN7^m_krSbHc2yz(W8()dZGC@)a4toW3>TT<2hpYWo?7@C+0{FVdbSI7Hq-|b;*iE z7t>+7Q@};_v`pxa8jbitT;I&Eqr(DBt(rF#|Y9tMUAJ8Wvn1-XrCdF;!$IW zgBKp#t9B2I;Ht)~cml+X8tYr1!V{mdHoON!jTzSoMm3|x+|@>BA!^jRL%`^$anfCI z-SFGf?zdpwCO#wpQliH8n}Vx$=k6LiL)2LDj38AtYETDQ*0^;Ch#JFk2yEq1BmI;N zzRBm1ecfpgHF~`#fK`eb!3u&qye=dj(uEe$`3u1mZ4oukV2jv#@`i`E`wJH!0^Bt@B5?pmh#Uy_O73CQ*T?XSYu3f^fXD@?P2LTav z(Z7i->D`}mN&e5JqNy{lYIJERcvXT|$#_+;GCf0(I4de@98lVe6Yuk~ zS7#t<9E&F)l|_xBnCJLG>~wSQSFqb~oxn+n8bwcPB1z?qbA5;!#`g%)KSd3%!1&=1 zHS``4Nb#tlUFMF*wyg1_kq|X*ydXf#sB!2+1fKX63s>}nsIfPfU{o_|bXUDu2vNiQ zBLSnMhOOpK+?7T<&((%?3;0F=q(qH9clIDDwL_~wh#C<;2~t&~26cd)N}l*Z)R-r` zQSz~=JZg07r-V`CWT{0MM2)1n1h7g`!$kcZ_L-*B@7Y6(aA-s@MO(x+L5ooQYqqT7U5KvigCp zHJq1@fdoYKylC9Ut+mK&(Oq!av2J2E=i+&f;WFa2;6iO$2=ZtJtKGI6k7%sMB!f;(`dG7rK>k=@YsAs!S<2k?g zu{+=rHG_bNx)i0Mi>IFHbXb=lzt7t_mx2t2%dif+&}ILmi|@ciZ6{IBcA>7EZL=-G zduV-=OYB_@-aaPoLF8H|> zr^NX}`8qJ3dzliD%Et556M~%FFQ;ob*uAzQa8kyzn(J>ouoj7bM!PslZc`l34q=LRFrEjVB!E>K&sJu#_@dwUiC+|;-MOD9n4&G>#uGe++MW$R z!gzK|ATZS%&!?0hEr!;a+9^pvy?hL0J!Q1cA3p@&Z0^0#8VF6svKLX$Zk^OU0j=$R z;QbAI1`-g_^OEos&&#II0jA)xHbBRib9wlJ;c{*GUK|6TKMeW~F4KdFdbZ1Z;X{S> zM@969HTFGBKtx@_U!cpBi&r0k%b(Vo*EpA?Y=%qs$6L@vSr}x zj+~5F&aQPPAfhhw-l0p8)v$Na)IENjisM|eK1hukueR#r$!&M}8U9V$>$PGeB+iP8 z8o@t&aN^B>+OP*ijW=}(NM%tYabr3j(}eY7nt|N|6#^$EY7{LOyyz(4#xg^Q8Z%lD zq<@MUPF~9fK-5UoB#`1!Big|pEf@b8`VpeW!%hT<88ucn3c?fLV$0%Sh#I$g5R7U@ zjVoT;W#}5JT}QXnQJ#;9~rH4aQ=QAX|4ZE$6q#I%p~gBt&@6P zjfTwQ-Edx}&nF{27!&nX zbg6N6(>8EvI-h`uy6AmImsK^pe+C!p#3`XwW(QZK0vt6ir z0t_}x!10`;PC!ImWJ=Lx&iK~~;4-br&8{uW_t@88440=XwxP@GZbtpVC8{-1&vv1% zYV3dA9Dj7Us7*jbUGjgSi~EPTrO?#3_nEnxbJ^)$Q`(OX$0K4ZtCH=ux=OD5dbMs!yz*f_xBey!=(^49&IE@RgD_d z0e)+Dz79l<=&b~{@~ENpp$kThby3ax!BveXy9r>GqQ;D;?J#QWZMOt3;vP6aFhyHL z6PZ!)K3Tgyt85`^_=Xdh>P3yVuYT2m*15y-b2Qg(^4Hd+AF9;&vKd=v!tDA=&}8n~ z5%uiWNnO?0k{^Ikqk$^{5j`*Yg7f0L)+7>KUadM<$hl;$XSnpvD>e()@X9}(aL_Ka&7hJ3j35cjmsw}#s{|LSYE~ma63*}rAH!)liA8bdLH~Xz) zU|ke`5%p{r>hG6rux*eAF3GizOAbQRC9w{=*tt8pfs0W@WQ*42>k_+_;ZnR$u*bZ9 z&%?h$-ZzG*XS+}jyXuVZ838VJ;s}VSORPM)EPmTX1Dbkd;2s0c<>fZ1QDgPM4!G%y zKHkoT73vyFkT@$UYMh+18YkYN^*xOtYD|tKAeBXp;w^XZn2ro=+8yjvV+fp-s4+0P z01r%W^3Y2VH4eoQq<@MUwQ8)u2Sn8q38Z+`P+n+_mX>;_wIOPpzDQwJ7Ux*)KY#8 zEyDT_!4z!~rV7|1reEr}6rzS+CyKaG^eY z|ML3KWXAs_>e;RH$_2sw4Z~LLz6Iw+t#+>DAVkkglp>y&l;Q;&z$L-tVkYMjvYX*D z!Nea&T6**t{6p`43yFHR3w2LGt2G@vfJ>e&0TFcxQ9>8Px?{4yWp0njEj7wVTEISr zOPw~p=yG3sI^MFkaWGNOcF}VGiG575s6lgZnKGV$h`I#SM;HGi-iyGc!Kb5soQwAX zhRfN-0q9~{nz0b}*qKtIp6x=7=c#@&%fZE^R;1)0L|wd<(IsZch#+vuospZyxs-)5 zTwdAq3O z0eAe-&ptGM7eoz>a|ENBQKQuD2i~jE;W7cEqsD}(f}g>6mt8su>vrV^0gw_klykE% zwD{T^#6#4uxJ!_#8a1c`9RF$F9EckG(g|$kQRDNJ;TSbKWPP0qQDgLT0$8P}u{mxa zMvbDdUcS&GqTUcp(H1ed5w?iEkHQ8*)JQKNFx871C%hZfgx2}gD1JB(gOQuqjc`cAUR2DU?p4`A=a-9*c40aw* z37nLuF*YI_4{X@r!$Tly)O$^k{wZoSSax3zqK4yp0x2Ff&MlmamhBZ~%OGlae3M<;wfT+=4g8)`3 zYM3b-Vbt)Izn=%~uCxQe6m1c61ude{dCxqE8XdY4nCeB12FfN~p>;m%?y0C*J_b(5 zF`zvjJ6aJf02sAs!SQA5e+<`P)rQPT;CsEf7+y0l47lmVB^K^?Ae zE~TEl1`xGZfxLA_o1 z9`jFPxLkg@8zW##{}CO*WoUb%p6&8Qc)M=d);aj2Lsl08BI=^t4qc8OY1RW8;d=Qs zZk)@L+fv8#!hK4(>5X;w9R#bHjR+EFMU7`0&sjJnUUxDrh4Fk|oq$v}o>#ah;xQdh zju{PhKid*GDdYLll$Us5SAt5lU_4LnOpyL*JfCi*90}t&uP1>NAJ2)Zf;VS)Bznz+ z@tiz>05QjNy{+Ep*U%@UGmPip;RK_av&IuJdOS}H{D$LsqVbK7ux?&P z1VGAo?((4wNxS904uJ7|WEMfH>UgFOa9#MMW-y+E=M&h<$MeLfSva1xbb?pHc>ZBc z0IM{f2e+At@2=C`b;yTp4~d<)cJ;8VYlHSH5davZ3u|yd6Ct^^U~+N+8J;; zGHY)p=TdTy;WFon;GTdjefnBK!~AVQ)U#cN8U03=X1NRT;k~O%35cjmNe6U!SLfm% za2e`cZ(#fKkyh}4;qqnER$LeFdd*D1Mbm?*XS-1Mom_kKOcB;4XA=Ptbt&kGE^2kR z;#(<02i@Alxnw)Y%f4ldSq1Vq&3VHb4idGBXW zXzDVXJqK%*@3CDkrACcc17&g3mu^2D2Uc>s2@+>TMGfu46L8`go^G!NQDb)q0jVr% zd>N2{$8_2vGZO3+A_$z6s4-IO2_Be!{(~@x8hcI=q<@MU-q-gIgs5R~fk28!4aG%9 zXc@RX;Wk8#4Oa*dGirQE6-135#Z$&Y)X+^P7}bm#7t;5xhNzL4O2Fu-p&DERcjc<^ z;D)en4vz_dl&BH5vNn>QxhxtEQDgK=f>hP0K^@?zt}fvaHIj1)Y~@j7Y&|QC8k$FT zbc3kT>mvcIQq;I^yBMR!vBPzXp+!V|CzzrwVpmr@g?p?WZ6Ip+{vt5diy97fQwKup zRG-vi9M^7={%bdF6k99GlCQos87UagEzdbFgC^rOgQ#b>PU=C#xAjwe;JkD;BOs#Z zB}s5zGH;E-zfa+|amN|XB`$~I(saMzj(Y!L+P%SLQUFoUcA|@V^CoKb)5u*r* zs7rJYbO{c$KMF33-G49cP(ISa-ZNYdY!i%UBe|7NU|mN067_7C&x-_Cn;yCU=mRc^ zI|zuVOIR;-(K0Lc1s8v-wkJ84z(R&g+aZEI=65Hv4Yi##)P?IE6B|yxmq48oJo_KqOs0I)AUKbn%^ZK8JJpTFP))K12{T#?>A>99#@%6ZLEt>PH91!j2e?XIT*t zQ5Q!8bn(|cj}IYVEi=;WSU%G7e=}TKFBI&tjdNb(gS!@T($h0Qv*)G)ELf_p;#9LgQ zI}#94mpMbwrRR+2y`ZV9rO938T%LQ@l6HSXe|N#nhQ9YAJi#hIlOSh#CjJ5u|^L8Xp}; zc891D{)<40M~(C57piZ8S%U8ba07DCF)|BI+`H1iGBM*l8@R%TvRTdpMWa%?y_&r`&K|RLuLm0GGNZM12)qKK3`q zA5i=k5D-xp{ZZ(0v}66<;F4*+ym6=Ub&1->aFHw8gf1;S=i=XUDY7Bz*)G(T0CP8) zWLTF_7Xl*c(s>NJSfA;G>$11?o=Kcb$WDff!NK+D(oJuELs*w{Iz&C&<+hJqsI5M891yg9pmwF#92{MBcpC7 zgiQH5U>8IUxk3U`S=5*vb_R#ATM#u)$P-BMsNq~T3N2@L(0T_^V}ddPVn&U7(_PT-+&aZ~m~4Ae7vmCCi7fdBq56!injsFhCGI#c7C?}gU6b}muRZk^P_ zd-5wz^o8?s&5D4Co)^^#*gExEzrr6-+&_gYcP<}k-oXr)xqk#d2K_$pMGahLmJ#)A z7b*84Cy9rz#G$!iV zF4R2%4cq+I2baCA2#Bal*<^Gn)!n56E@K}gCvq-DhZrt$N`k8z^?o>R2bZreh6!EpK2L$EGx_w?}x$64`2 zJ==xaW3!iy41{%wzd=AmU0xfZ%k@>Hu{^b{>GJA@iHSRvtA9y*6Uhh>A|rhN!W>cB$lZSf!|O*CG(3MjH#YGH4N{ zN(581MU)6yL{j_g%Mdljs1lg!MU9lSE_nN)eb!JfuH8I7#%P_7Qw86i8#S;$1x;pl z0a4Fxo$nq9{_3Vu>j}N#ytow;5YhAUcsg#av`>-v)JECOqn|jJl#>h>$3nqXjmw`_ z-3J%bG@_pE@>%$eGkjCTVQ|sRBp{+LDKpUJ<>JwDFw!>7EgYa*KGG7-FkDVeT!T@= ze()Z!gU$%bfy`&Y&Iq!qAsWBpi4K8F%ICe*1+})=Mo;za50Mz+(zBY z^{G6#M2scs*)CKB41U+`F1R$CMnFVe!cEY{^P9myXzFp!2W7jK@3Hj%T%<&eE+a4D zuySqQ@(fs=T}O~OD=KQ-c%X)m7D_hi5H*T?2uNj7qtrD9qekGpsSUu+a2tV>5;dm$ zyov`lE4GU!M2*gS3DQ4BjdrDXHbT@0I7lGHqsGObL(%efvB3d|8t;!1AZFCKeZm&~ zY#bhG!Bq{@Qv{=$QKOmt*oF`_OwJQ9I%hNxkfN?-99&tSRyP4VmlOggWjs6iT*d?Q zIp!t{<9SR5LHeihe9TMV9mccD3j!%Vp6|98h?cjq?6$&qo|H>~nB&>I`y%vHy4a%` zjOUaO1f!bc`R1w5cq`?HZv>1U&xd_=a9673^{N5uw(A!GkTRYRZx-BI-zDtc7#PoU zWY5GD<98g838z+`zHkD!FbM6C4f~L&+;=4<9N0`>lz0w zLbfHr6m1cs7vf^|9+I#79!KHp20TFc>ZG$dXY+4)um!ECU>UA&QV{usw z7i$#_pVFU8pMoy>q7EYp>#a5)e_B0k-I3`O&ODH1%djx3A}1lCz~o zjaT);aac9JvUUrs(ALugiL;`j#=Vhs5YoD20siPPAdY}k7Bza#6GV+QsXOr}l|G3C zPD<1$PKn0@8`L=FDMXF*I|S*UqQ-|-jru^;xR_2L#iPdM-u*CYjMhU#3w^J+InIOCN7?{5S^O4Mk# zr8n;HEpi3(AZiRQBS=+^8q@(6-!JhP0K^38rX^n7j&G#H~ktu0zz=y_UdKFKWcP<>0sHjFy@fT)X*N%xImp zj|j$dk^F4@OO@@kh0@=_mK4b1wNm z87?Q?1@{E3KCg2W8fJbRQO|av?g>cUR$K#IWRnPps7wB8Tw~erYzr9A_d<=E_9`D~ znSU5A@+$e()zgmKonQP)A>60wnhh`MC2MHi3lt_Pv1e;nj4 z*Sma=g>ICUc7H>$S0E0nZeEML!AgHDLE@~asNrZ{G64Qs;bVgXmJl`GPbDCgMU6v6 zQ8PzWgVWvoLe7$K-6$NMPRBIHD3OChObwO zh-*8JYd5jJjMlk4d=a)zm;7aB&}3{=iF$VH)G{l<&rADw&lPZ90y7DS*m;r36r7iK z?Y=LEF)&T{r4Q#4wT0pG#Z&O3!_Tz2-@qlW4^hu{DHe`_YeQci1DEu{1Vq#&!voiN zYNHZ&a7j6BSF=y~7zo+Ua9N%sh)u;=o2G$FNCHvMcA;X^rS^@#fJ@|U0wU^?>WMB5 zm+i%ekca==WWc!u>|(g6#S8YB<@&bD;G$G#v!tHwqBx=m_t^7>%ka;788jduqAs`A zqf7mS&n~bo_nj5aaW39_87?uJX83uzS7=fTTujyx^=y|S;U06=mcy?sk9-J-s7s<3 zy38t5#J_4_YGQwrbBX%TrJ|`bpW5&a*^a|F(y-Yju$r=nAaPby)G(d+5u=8}+LtL1 zHHO&}kjkRQ*z}{g;MNy+;}0nBTnL<$sPW4C4BFk?q23XqhKUD3`lqOIr~jHa5H;TW z5=il=k+r!uTBhY|Z-%I$96*4WQDfqSx#+jsZSoMfsxcynU{o_|OnaVU4pBoPjDXQm zLoQA5X#9e4ac>}MltmH%DN$qpPQlZSEq=W@1ySS4DS}khs6icI_m%mn5H@)&X zy{OUef$eo@opshvkkKz6X=TBT)>+nQ8Me-+(*rG`$?V!p)U#VB^)3n*qj)CzT?yQ6_Z2uKP zKtx?)z0sxL@f*Lv#e8+cvz*K8!wi??K7y-FL7K*^z{P$eQO|avF4Tw0tauCSqPBy8 zh`OBGh%V0>&S?ZLC)yZx=v%(W9v@}6G#fJ?_gHqPkN9W3e(MnRY!~W&prM~SHw2fv zJqU=XOOy|~bUC#86u6Aj3|-8*q#S3soVA^YE(5Ngs}1YYT$iY4yHG#UmF-`f2rf(Y z35cl6VPAA9-P~#iG;(*ASG&S-7R>Sdr$WLZ4foiPbNrJjT+PeZXEKXFGLN) zSp>H7sIlU1G)9e$eUw5WYCJR}fK`ebM_VRi)L7UdItp4ufi=MtYZ305f)-Iyci}{c z8d`P)rg~AM$+qijpmi47hrj09O~NTg>(pK2g00hHkV#8uoocQ`J-c;M-_n+g#kR>e((-)JRg- z?gcJd{RoJt%brc>V%1ouA-dF_u!D06zrb+uxwsV9C8fP#8LZ3h`b0h3g}SP7{Kko< z;BuxZ0TFcx*o-chj@G>jF3&p@zT#YhE-_pNXw60!r*=Uy;1bfEsAs!SKL*_#`gtO_ zv^O9iqAr`apo@=68D5q*pMQBw|METd{)*J`d|Yk|Hh86VQ4PTAqzyshtf=v%K6a+xrtp@$vlqU@r_IDLa;{!g%f#NPw8*`C6#p@2NdKdOi`xvwjG{sOEU?KHumy zjOW_n1dJZfomLp)t{l5%;4oOXNhb(^l<~Z6|4bxxF@3lR#`BJ|1gWaynL5C3A!9zn zc&>Aaz*atQj_k)_^9{^D9x$Zk^O^ZfYTo z7D4MwDmNQO>e(*T9rX=o zzr_b<2W1fuQI~aFagC?G`yB`Cl6z`YodM-zpd^{$GRaYJf5X{vb!)=9Oo=Dz*)AW2 z7q|1|*WtzOD>n#;sEg}1bXlMKPzPKR21L!_Tng?mTv{|2yd%W2zu{GI**%)5XS+~W zHEQ>WiH1Ga!;pZ8x-8$0E;E)NuLmxlwVR&fT(a&nTy%a2p4xCp+^-ETHrIc~e2?vXC^c$?UD=GoxZmIeC9rZ?L6A5r zDr%I32%?6!iS1H|8cp2@NM%t&VW%K!n6-S{4(y7&2%MCtL0t(*aLO@)s4>rSe3bI`?;5H<3`2}U)e#`N|l z=Rwri8BM_Gs4;uA;B7C4t6QhRy1AYw08*mHfDpk$_MwKhh7dJgUM5IYjT+Pe_WihQ z4@8aAHwkRzQA0*44x>f`l}{lMHM*q|z$!(JxVtYfYFxf`9NXQj#{^TfMbzDaEuyFD z3?qmdJ+cT)^`b`a9Z@Zzbq08MZq2ouhv|&g=`vK%I#pFw@vW3jTZwvh>!kiDw#h?n zZD^fGcM}lN^Rh^AUcQ7U$%BhizdI{9m!u~QmtTvV@f+v1k1r-e!+bJ~sAs!SPdbDT z@1zMX0pkgXsEfr;T;q$=wjKu;O}zt;IG4ER442|of**srhsoH3OPhW~J==x4P+yo) z*a2J|hZ7J{7n5D+607(OAKaaM(_MRD`ACa?#c;VCBlywbr0keFnJPj3pqVE>m}-%LIe(j+X+qXu<=v2A{r zLe!Y%Ltrb98hPRP^W>C~$xvkMoXB{7>Fi+JK>1@k zUznF)T}X)3yr`{X=0!W|YbRjQ`tk01%Yl~W*UDj;U%4AucE-v)f#ua%GM>*OdVn&u zMExPKL`IVkNtRXTaE*6&y&nuL{zX0^0*m~24vR%CbGIqM*&ENIaJWmx^I1gNlyK*0 zFR%pGkPt~0tMkaRPE9@&SQ;;wXjl!bOJ^sC z9El?#k}O8y$dXc$+X6*B@xH#Vz>?piQZ;%$U{u5RrT6DRb>}fj5~xI~QJK|<4^Xn_ zk8J?ekhPGI!BoTl3Zoi{Nr`wPKD3R*sZb653(Wfshxc4>1J$tVCQ0w48is)}&7d0N zMi{)EdTG^I)x@ZVY|IiFsD|rk62w&v(;VjghK+AVwt#Bv(;^weRE<(y^*B(C4`n1w ztQvXG7(sB78!v~e^4~K_fC|+Jj$zJ?$vbkr3sht5Jd!k2)e!x_`fF!)gK9ijNMZ+9 z4UHqks2XoJD_?_Zq?wSwA*sfeJs_~?9GgblHO51ku0HT7e}&si|t$M%i&C-h!VeH+un82O~+6KqXR*-qlU08jtHe zWS|<~#gmZ1RHJnQvq3cH8^vXyJC#b}RH(*xh0I-zDGmp6KsA&(B zWvCjZrg<+xHTpYA;E+@!Ah{J)Bf{VQ43xX9K9VU_BIJ%tiMZ8qZWE}+3Ju4%C1QB0 zVPtw@CA0zekA9E~UY;x_AyV_A zHZdA6Eu+c_rob}MDz;l-2?*e^9Aq8_^%;%+C$QwMA>;WhqB9RZUFNwOSgvm%A(AX( zVvr?D8Ic7n&-1j6R}8c?MF@vw{faE-M`s#J|fE{tk?aA;d3P;H1MNdlEfHL@2sqiW=T>No>bV{!@!8B8^L z*D|Vc*V#h>blTY@PK9cSHi(|kzi9%hVUaOx^fLUqV`DfO&o7;#^Bd0l74}0Lm=;GuBX8$*^F0q~@M3SXB5m`$AlDQFBG7p3}3oHex z92WBz%tN}pn|<)Z>kV=;p3fqBuvOK40}sIdhaU-%WNAo3mWk%+!B7YtjlU=pSd~mVXYJIO2x!lNwj^Y*_Wb+T$$0B0v?ebT=th4= z;#9Qf0=I1ZPr0cm$_(1GY!gX(r}iAAnX3iux!r|CO1J0PzRW`h!=}Am(4Nn3Cqdly zY^Y?;Z@8P~xDDE~|2~p2%=Y|rT1E`C=jQK8n7BQ=Y+;&edU0)vOM zLNlv37!U3F4=<85)b=d;fnB;+&7nP?P?Ffe+w-67YOp;wEZl^j!deqZ0*BO|eZtkP z;oLP>_t8F3?iPiSOsNtfzjX;qglyN})uBBbMw6J~wddDPH}ar#PA*TY7RpWL|I3Z& zTN_`UWd3{ZsP$}vB4bod#`8<3Xm>z?Vwn!iOI1Ayk(w8^N6fs0nFQhPfSeg-x~m3S zTKp9bi~eOsHE!!^=L1Wf9vRPP5xr6Goaui7Sbi`jA(AZjl5ve2H_P#a&Bkk*J_5`6 zpE)esG?|ke4L=!$r&6vfCgb@mqED3B8mzDb7S~D=BFR#bf-GA*Qt_kMUurz55?D^> zb6B)4GKYCr^tjD{bupSq#`9T3+7xHurU5MDwMmF1OGzrSST2}=@12yF?^tR(aE%== zu?8_>%Wo^NtPRF$go%xoGyh9shF8x@CoMJpA2J&F2><{9 literal 0 HcmV?d00001 diff --git a/logs/self_tuning/ademamix_golden/study_1/librispeech_conformer_pytorch/trial_1/flags_0.json b/logs/self_tuning/ademamix_golden/study_1/librispeech_conformer_pytorch/trial_1/flags_0.json new file mode 100644 index 00000000..b20c49c7 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/librispeech_conformer_pytorch/trial_1/flags_0.json @@ -0,0 +1,62 @@ +{ + "logtostderr": false, + "alsologtostderr": false, + "log_dir": "", + "v": 0, + "verbosity": 0, + "logger_levels": {}, + "stderrthreshold": "fatal", + "showprefixforinfo": true, + "run_with_pdb": false, + "pdb_post_mortem": false, + "pdb": false, + "run_with_profiling": false, + "profile_file": null, + "use_cprofile_for_profiling": true, + "only_check_args": false, + "op_conversion_fallback_to_while_loop": true, + "delta_threshold": 0.5, + "tt_check_filter": false, + "tt_single_core_summaries": false, + "runtime_oom_exit": true, + "hbm_oom_exit": true, + "test_srcdir": "", + "test_tmpdir": "/tmp/absl_testing", + "test_random_seed": 301, + "test_randomize_ordering_seed": "", + "xml_output_file": "", + "experimental_orbax_use_distributed_process_id": false, + "experimental_orbax_use_distributed_barrier": false, + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "workload": "librispeech_conformer", + "tuning_ruleset": "self", + "tuning_search_space": null, + "num_tuning_trials": 1, + "data_dir": "/data/librispeech", + "imagenet_v2_data_dir": null, + "librispeech_tokenizer_vocab_path": "/data/librispeech/spm_model.vocab", + "framework": "pytorch", + "torch_compile": true, + "experiment_dir": "/experiment_runs", + "experiment_name": "submissions_a100/ademamix_golden/study_1", + "save_checkpoints": false, + "save_intermediate_checkpoints": true, + "resume_last_run": null, + "append_timestamp": false, + "use_wandb": false, + "profile": false, + "max_global_steps": null, + "overwrite": true, + "hparam_start_index": null, + "hparam_end_index": null, + "rng_seed": -486301922, + "set_pytorch_max_split_size": false, + "pytorch_eval_num_workers": 0, + "capture_jax_trace": false, + "skip_evals": false, + "?": false, + "help": false, + "helpshort": false, + "helpfull": false, + "helpxml": false +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_1/librispeech_conformer_pytorch/trial_1/measurements.csv b/logs/self_tuning/ademamix_golden/study_1/librispeech_conformer_pytorch/trial_1/measurements.csv new file mode 100644 index 00000000..081cacd5 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/librispeech_conformer_pytorch/trial_1/measurements.csv @@ -0,0 +1,322 @@ +global_step,grad_norm,loss,train/ctc_loss,train/wer,validation/ctc_loss,validation/wer,validation/num_examples,test/ctc_loss,test/wer,test/num_examples,score,total_duration,accumulated_submission_time,accumulated_eval_time,accumulated_logging_time,preemption_count +0,0.4999999105930328,33.852783203125,,,,,,,,,,,,,, +1,,,32.56430697422715,1.068215187960184,31.449624522804903,1.2736831941292908,5348.0,31.58821859474805,1.2729876302480043,2472.0,10.965123414993286,78.10448575019836,10.965123414993286,66.33343935012817,0.0,0.0 +1,0.4999998807907104,33.20514297485352,,,,,,,,,,,,,, +2,0.4999999701976776,33.73041915893555,,,,,,,,,,,,,, +3,0.4999999701976776,33.86479187011719,,,,,,,,,,,,,, +4,0.4999999403953552,33.27867889404297,,,,,,,,,,,,,, +5,0.4999999701976776,33.576141357421875,,,,,,,,,,,,,, +6,0.4999999403953552,33.6898307800293,,,,,,,,,,,,,, +7,0.4999999701976776,32.65314483642578,,,,,,,,,,,,,, +8,0.4999999403953552,33.03567123413086,,,,,,,,,,,,,, +9,0.5,32.67837905883789,,,,,,,,,,,,,, +10,0.5,32.64419174194336,,,,,,,,,,,,,, +11,0.4999999403953552,32.788238525390625,,,,,,,,,,,,,, +12,0.4999999701976776,33.0091667175293,,,,,,,,,,,,,, +13,0.4999999701976776,32.57346725463867,,,,,,,,,,,,,, +14,0.4999999403953552,32.43279266357422,,,,,,,,,,,,,, +15,0.4999999701976776,31.562088012695312,,,,,,,,,,,,,, +16,0.4999999701976776,31.934202194213867,,,,,,,,,,,,,, +17,0.5,31.34388732910156,,,,,,,,,,,,,, +18,0.5,30.934812545776367,,,,,,,,,,,,,, +19,0.4999999701976776,29.915504455566406,,,,,,,,,,,,,, +20,0.4999999701976776,28.5269832611084,,,,,,,,,,,,,, +21,0.5,27.409582138061523,,,,,,,,,,,,,, +22,0.5,26.401222229003903,,,,,,,,,,,,,, +23,0.4999999701976776,24.07742691040039,,,,,,,,,,,,,, +24,0.4999999701976776,22.48136329650879,,,,,,,,,,,,,, +25,0.5,20.53371810913086,,,,,,,,,,,,,, +26,0.5,18.51685333251953,,,,,,,,,,,,,, +27,0.4999999701976776,16.58028221130371,,,,,,,,,,,,,, +28,0.4999999701976776,14.544387817382812,,,,,,,,,,,,,, +29,0.5,12.655977249145508,,,,,,,,,,,,,, +30,0.4999999403953552,11.01361846923828,,,,,,,,,,,,,, +31,0.4999999701976776,9.644628524780272,,,,,,,,,,,,,, +32,0.5,8.566202163696289,,,,,,,,,,,,,, +33,0.5,7.828676700592041,,,,,,,,,,,,,, +34,0.4999999701976776,7.37141752243042,,,,,,,,,,,,,, +35,0.4999998807907104,7.186089992523193,,,,,,,,,,,,,, +36,0.4999999403953552,7.184430122375488,,,,,,,,,,,,,, +37,0.4999999403953552,7.254039287567139,,,,,,,,,,,,,, +38,0.4999999701976776,7.284192562103271,,,,,,,,,,,,,, +39,0.4999999701976776,7.264735221862793,,,,,,,,,,,,,, +40,0.4999999701976776,7.198292255401611,,,,,,,,,,,,,, +41,0.4999999403953552,7.10597562789917,,,,,,,,,,,,,, +42,0.4999999105930328,7.024555206298828,,,,,,,,,,,,,, +43,0.4999998211860657,6.981740474700928,,,,,,,,,,,,,, +44,0.4999999403953552,6.971043109893799,,,,,,,,,,,,,, +45,0.4999999403953552,6.956645011901856,,,,,,,,,,,,,, +46,0.4999999105930328,6.904799938201904,,,,,,,,,,,,,, +47,0.4999999403953552,6.872978210449219,,,,,,,,,,,,,, +48,0.4999998211860657,6.810488224029541,,,,,,,,,,,,,, +49,0.499999850988388,6.787607669830322,,,,,,,,,,,,,, +50,0.499999850988388,6.757025241851807,,,,,,,,,,,,,, +51,0.4999997615814209,6.687803268432617,,,,,,,,,,,,,, +52,0.499999850988388,6.66774845123291,,,,,,,,,,,,,, +53,0.4999999403953552,6.652022838592529,,,,,,,,,,,,,, +54,0.4999999403953552,6.618642807006836,,,,,,,,,,,,,, +55,0.4999997019767761,6.577996253967285,,,,,,,,,,,,,, +56,0.499999850988388,6.5482177734375,,,,,,,,,,,,,, +57,0.4999998807907104,6.520592212677002,,,,,,,,,,,,,, +58,0.4999997317790985,6.48127555847168,,,,,,,,,,,,,, +59,0.4999998211860657,6.464359760284424,,,,,,,,,,,,,, +60,0.4999999403953552,6.448277473449707,,,,,,,,,,,,,, +61,0.4999997913837433,6.41874885559082,,,,,,,,,,,,,, +62,0.4999996423721313,6.39103364944458,,,,,,,,,,,,,, +63,0.4999998807907104,6.359578609466553,,,,,,,,,,,,,, +64,0.4999997615814209,6.33571720123291,,,,,,,,,,,,,, +65,0.4999997913837433,6.337650775909424,,,,,,,,,,,,,, +66,0.4999995827674866,6.301974296569824,,,,,,,,,,,,,, +67,0.4999998807907104,6.2760910987854,,,,,,,,,,,,,, +68,0.4999997913837433,6.281323432922363,,,,,,,,,,,,,, +69,0.4999996423721313,6.276308059692383,,,,,,,,,,,,,, +70,0.4999997019767761,6.237772941589356,,,,,,,,,,,,,, +71,0.4999999105930328,6.239442825317383,,,,,,,,,,,,,, +72,0.499999850988388,6.205280780792236,,,,,,,,,,,,,, +73,0.4999997019767761,6.200094699859619,,,,,,,,,,,,,, +74,0.4999996125698089,6.202697277069092,,,,,,,,,,,,,, +75,0.4999998807907104,6.181275844573975,,,,,,,,,,,,,, +76,0.4999998211860657,6.172267436981201,,,,,,,,,,,,,, +77,0.4999998211860657,6.157735347747803,,,,,,,,,,,,,, +78,0.4999996423721313,6.159414768218994,,,,,,,,,,,,,, +79,0.499999850988388,6.156438827514648,,,,,,,,,,,,,, +80,0.4999999105930328,6.134679317474365,,,,,,,,,,,,,, +81,0.4999995827674866,6.112738132476807,,,,,,,,,,,,,, +82,0.4999994039535522,6.102005958557129,,,,,,,,,,,,,, +83,0.4999999105930328,6.117875099182129,,,,,,,,,,,,,, +84,0.4999997913837433,6.10420560836792,,,,,,,,,,,,,, +85,0.499999850988388,6.064448356628418,,,,,,,,,,,,,, +86,0.499999850988388,6.074485778808594,,,,,,,,,,,,,, +87,0.4999998211860657,6.051887035369873,,,,,,,,,,,,,, +88,0.499999850988388,6.068389892578125,,,,,,,,,,,,,, +89,0.4999996423721313,6.06727933883667,,,,,,,,,,,,,, +90,0.4999995529651642,6.035016059875488,,,,,,,,,,,,,, +91,0.4999999105930328,6.0482096672058105,,,,,,,,,,,,,, +92,0.499999850988388,6.032508373260498,,,,,,,,,,,,,, +93,0.4999995231628418,6.01521110534668,,,,,,,,,,,,,, +94,0.4999995827674866,6.0053815841674805,,,,,,,,,,,,,, +95,0.4999999105930328,6.022454261779785,,,,,,,,,,,,,, +96,0.4999999105930328,6.0197930335998535,,,,,,,,,,,,,, +97,0.4999998211860657,5.98431921005249,,,,,,,,,,,,,, +98,0.4999996423721313,5.985709190368652,,,,,,,,,,,,,, +99,0.4999999403953552,5.993132591247559,,,,,,,,,,,,,, +100,0.4999998211860657,5.993191719055176,,,,,,,,,,,,,, +500,0.4999997317790985,5.733994960784912,,,,,,,,,,,,,, +1000,0.4999994337558746,3.4695403575897217,,,,,,,,,,,,,, +1500,0.4999994039535522,2.7535665035247803,,,,,,,,,,,,,, +1616,,,4.018316224717581,0.765940448686114,4.212096738748242,0.7391976053686091,5348.0,3.992824133815085,0.7336542562915118,2472.0,1757.0212490558624,1870.887146472931,1757.0212490558624,110.83208703994752,0.0343098640441894,0.0 +2000,0.4999993443489074,2.3062052726745605,,,,,,,,,,,,,, +2500,0.4999993145465851,2.2095000743865967,,,,,,,,,,,,,, +3000,0.4999993145465851,1.9899390935897827,,,,,,,,,,,,,, +3500,0.4999991655349731,1.8512542247772217,,,,,,,,,,,,,, +3833,,,0.7888560762362172,0.2638028199425647,0.9988812796677216,0.3018104572007917,5348.0,0.7281453642170462,0.2421952755265777,2472.0,3503.4649250507355,3665.88813328743,3503.4649250507355,156.67479920387268,0.0733029842376709,0.0 +4000,0.4999992251396179,1.8986825942993164,,,,,,,,,,,,,, +4500,0.4999992549419403,1.81688129901886,,,,,,,,,,,,,, +5000,0.4999992549419403,1.6853516101837158,,,,,,,,,,,,,, +5500,0.4999990463256836,1.7224372625350952,,,,,,,,,,,,,, +6000,0.4999991953372955,1.62224543094635,,,,,,,,,,,,,, +6123,,,0.4452975969018467,0.1541953663118347,0.668249043255224,0.2019601216627239,5348.0,0.4199425920623911,0.1415920216115207,2472.0,5250.175142049789,5459.857209920883,5250.175142049789,201.2211222648621,0.1294331550598144,0.0 +6500,0.4999991059303283,1.5863786935806274,,,,,,,,,,,,,, +7000,0.4999991357326507,1.5850377082824707,,,,,,,,,,,,,, +7500,0.4999991059303283,1.5983829498291016,,,,,,,,,,,,,, +8000,0.4999991059303283,1.5255777835845947,,,,,,,,,,,,,, +8500,0.4999991357326507,1.4506934881210327,,,,,,,,,,,,,, +8655,,,0.3602500813857133,0.1267030855267419,0.5876604156463482,0.1749336165693043,5348.0,0.3605698855571327,0.1206710945910263,2472.0,6995.479728221893,7252.397528409958,6995.479728221893,245.47307872772217,0.2418584823608398,0.0 +9000,0.4999990463256836,1.4994120597839355,,,,,,,,,,,,,, +9500,0.4999992251396179,1.5512374639511108,,,,,,,,,,,,,, +10000,0.4999991953372955,1.4476759433746338,,,,,,,,,,,,,, +10500,0.4999989867210388,1.4455174207687378,,,,,,,,,,,,,, +11000,0.4999992847442627,1.4769577980041504,,,,,,,,,,,,,, +11059,,,0.3148550858672123,0.112921857793708,0.53931664759456,0.1607299763433592,5348.0,0.3231082700900058,0.1078951110027826,2472.0,8741.733561992645,9046.708485126495,8741.733561992645,290.6490111351013,0.3359074592590332,0.0 +11500,0.4999991357326507,1.4428976774215698,,,,,,,,,,,,,, +12000,0.4999991655349731,1.3563435077667236,,,,,,,,,,,,,, +12500,0.499999076128006,1.441177487373352,,,,,,,,,,,,,, +13000,0.4999992549419403,1.4758919477462769,,,,,,,,,,,,,, +13355,,,0.2846758628999526,0.1035454408049575,0.5182867067416366,0.1547433978660744,5348.0,0.3071018190689722,0.1025734771393171,2472.0,10488.446098566055,10841.1125998497,10488.446098566055,334.5619010925293,0.4564213752746582,0.0 +13500,0.4999992251396179,1.37811017036438,,,,,,,,,,,,,, +14000,0.4999990463256836,1.3304669857025146,,,,,,,,,,,,,, +14500,0.4999991655349731,1.4251303672790527,,,,,,,,,,,,,, +15000,0.4999994039535522,1.3908041715621948,,,,,,,,,,,,,, +15500,0.4999992251396179,1.3445260524749756,,,,,,,,,,,,,, +15907,,,0.257056379160184,0.0942445965495649,0.4925074777570574,0.1475401921498575,5348.0,0.2902662318939931,0.0983486685759551,2472.0,12233.473339557648,12634.514830112455,12233.473339557648,379.58304166793823,0.8120191097259521,0.0 +16000,0.4999993145465851,1.4226760864257812,,,,,,,,,,,,,, +16500,0.4999991357326507,1.3534446954727173,,,,,,,,,,,,,, +17000,0.4999991059303283,1.3157155513763428,,,,,,,,,,,,,, +17500,0.4999991655349731,1.3212980031967163,,,,,,,,,,,,,, +18000,0.4999990463256836,1.3275620937347412,,,,,,,,,,,,,, +18300,,,0.2425476053575052,0.0895698831861464,0.4747996055291842,0.142287452324627,5348.0,0.2783933640880057,0.0924786220624378,2472.0,13980.496501922607,14430.19681930542,13980.496501922607,424.8829026222229,0.9918942451477052,0.0 +18500,0.4999989867210388,1.3186323642730713,,,,,,,,,,,,,, +19000,0.499999076128006,1.297757625579834,,,,,,,,,,,,,, +19500,0.4612111449241638,1.280515432357788,,,,,,,,,,,,,, +20000,0.4999992549419403,1.2748914957046509,,,,,,,,,,,,,, +20500,0.4999993145465851,1.3519576787948608,,,,,,,,,,,,,, +20592,,,0.2238051361783805,0.0831623950078811,0.4520509676543349,0.1334234538695505,5348.0,0.2681596897380476,0.0874616618934454,2472.0,15726.650465726852,16225.020930051804,15726.650465726852,469.6380949020386,1.3005595207214355,0.0 +21000,0.4999992549419403,1.3740042448043823,,,,,,,,,,,,,, +21500,0.4881057739257812,1.2342078685760498,,,,,,,,,,,,,, +22000,0.4999991953372955,1.2677001953125,,,,,,,,,,,,,, +22500,0.4999989867210388,1.3418766260147097,,,,,,,,,,,,,, +23000,0.4999991953372955,1.3029661178588867,,,,,,,,,,,,,, +23169,,,0.2137061732648988,0.0779640705633407,0.4544447426913803,0.1325544344131704,5348.0,0.2625980658026324,0.0869741839822883,2472.0,17472.34481048584,18019.78716826439,17472.34481048584,514.7942316532135,1.469043493270874,0.0 +23500,0.4999990463256836,1.302031636238098,,,,,,,,,,,,,, +24000,0.4228472113609314,1.252861499786377,,,,,,,,,,,,,, +24500,0.4999994039535522,1.2404673099517822,,,,,,,,,,,,,, +25000,0.4999991059303283,1.2718812227249146,,,,,,,,,,,,,, +25500,0.4999991357326507,1.190868854522705,,,,,,,,,,,,,, +25545,,,0.2012373931416153,0.075707684667372,0.4330134438730661,0.1277844831748177,5348.0,0.2519238136331376,0.0829728027948733,2472.0,19217.717973709103,19814.8000998497,19217.717973709103,560.9092576503754,1.828929662704468,0.0 +26000,0.4980729818344116,1.2635219097137451,,,,,,,,,,,,,, +26500,0.4999991357326507,1.280403733253479,,,,,,,,,,,,,, +27000,0.4999991059303283,1.283491134643555,,,,,,,,,,,,,, +27500,0.4999992549419403,1.180885672569275,,,,,,,,,,,,,, +27875,,,0.1893225379236285,0.0712812817135577,0.4232125982645168,0.1242601264906097,5348.0,0.2433782018194722,0.078402697377775,2472.0,20962.9728038311,21607.65250182152,20962.9728038311,604.724237203598,2.0684447288513184,0.0 +28000,0.499999076128006,1.1951285600662231,,,,,,,,,,,,,, +28500,0.4999993741512298,1.2130359411239624,,,,,,,,,,,,,, +29000,0.4999992251396179,1.1237852573394775,,,,,,,,,,,,,, +29500,0.4277617335319519,1.237488865852356,,,,,,,,,,,,,, +30000,0.4999991357326507,1.3006877899169922,,,,,,,,,,,,,, +30432,,,0.1802439880267875,0.0678589165029257,0.4170667001802039,0.1208323275237773,5348.0,0.2367694337134653,0.0762090467775678,2472.0,22708.710742235184,23403.356159448624,22708.710742235184,650.1285529136658,2.2313308715820312,0.0 +30500,0.4999992847442627,1.2492752075195312,,,,,,,,,,,,,, +31000,0.4999992847442627,1.2939578294754028,,,,,,,,,,,,,, +31500,0.464594691991806,1.227510690689087,,,,,,,,,,,,,, +32000,0.4999991953372955,1.2245360612869265,,,,,,,,,,,,,, +32500,0.4999992549419403,1.144282579421997,,,,,,,,,,,,,, +32790,,,0.1686431284034025,0.0642908038779608,0.4079771833245429,0.118881861632791,5348.0,0.2304708922753081,0.0751731562163589,2472.0,24454.32293319702,25198.224741458893,24454.32293319702,695.7054653167725,2.575826644897461,0.0 +33000,0.499999076128006,1.1167945861816406,,,,,,,,,,,,,, +33500,0.4342051446437835,1.1685550212860107,,,,,,,,,,,,,, +34000,0.4284201860427856,1.0732322931289673,,,,,,,,,,,,,, +34500,0.4999993443489074,1.154900074005127,,,,,,,,,,,,,, +35000,0.4999991655349731,1.168883204460144,,,,,,,,,,,,,, +35167,,,0.1633243077986538,0.0618454861485975,0.401488988016752,0.115618210785497,5348.0,0.227936026114588,0.0728982592976255,2472.0,26199.73352622986,26990.98868870735,26199.73352622986,739.260491847992,2.649347305297852,0.0 +35500,0.4994716942310333,1.1196967363357544,,,,,,,,,,,,,, +36000,0.4999991059303283,1.1278778314590454,,,,,,,,,,,,,, +36500,0.4912120699882507,1.0869452953338623,,,,,,,,,,,,,, +37000,0.4999991655349731,1.1595370769500732,,,,,,,,,,,,,, +37500,0.4999991059303283,1.1831625699996948,,,,,,,,,,,,,, +37696,,,0.15306217709954,0.0585202858808541,0.398219505035664,0.1136580891227731,5348.0,0.2228330255177753,0.0707046086974184,2472.0,27945.150611400604,28787.031717062,27945.150611400604,785.2804026603699,2.9802584648132324,0.0 +38000,0.4999991357326507,1.1297125816345217,,,,,,,,,,,,,, +38500,0.4999992847442627,1.0635303258895874,,,,,,,,,,,,,, +39000,0.4999993741512298,1.1665308475494385,,,,,,,,,,,,,, +39500,0.4999990463256836,1.1299757957458496,,,,,,,,,,,,,, +40000,0.4999991953372955,1.1938154697418213,,,,,,,,,,,,,, +40029,,,0.1467357326616722,0.0564528318182799,0.3883773561507936,0.1106744556558683,5348.0,0.2162197712352409,0.0689781244287368,2472.0,29690.69269633293,30579.836644887924,29690.69269633293,829.5948066711426,3.313499689102173,0.0 +40500,0.488642930984497,1.1234573125839231,,,,,,,,,,,,,, +41000,0.4999992847442627,1.0686043500900269,,,,,,,,,,,,,, +41500,0.4999993741512298,1.091375708580017,,,,,,,,,,,,,, +42000,0.4999996125698089,1.2021406888961792,,,,,,,,,,,,,, +42457,,,0.1373192074511263,0.0533489516982272,0.3837925943716094,0.1090426302322213,5348.0,0.2129698387799213,0.0666829159303719,2472.0,31436.16641187668,32374.587361335754,31436.16641187668,874.6922862529755,3.3926544189453125,0.0 +42500,0.4785641431808471,1.036168098449707,,,,,,,,,,,,,, +43000,0.496736615896225,1.0762133598327637,,,,,,,,,,,,,, +43500,0.499999076128006,1.1371444463729858,,,,,,,,,,,,,, +44000,0.4999994337558746,1.1237337589263916,,,,,,,,,,,,,, +44500,0.4999992549419403,1.1159050464630127,,,,,,,,,,,,,, +44951,,,0.1291461132508624,0.0502072852114956,0.3754650052350186,0.1060300294501038,5348.0,0.2074357619846441,0.0668047854081611,2472.0,33182.07935786247,34170.63980221748,33182.07935786247,920.2076592445374,3.678757905960083,0.0 +45000,0.4999992251396179,1.0724599361419678,,,,,,,,,,,,,, +45500,0.4999991059303283,1.145480036735535,,,,,,,,,,,,,, +46000,0.4832265973091125,1.0957775115966797,,,,,,,,,,,,,, +46500,0.4890971779823303,1.1138497591018677,,,,,,,,,,,,,, +47000,0.4999991059303283,1.1426126956939695,,,,,,,,,,,,,, +47271,,,0.1227502927771764,0.0487552091205493,0.3674541563065099,0.1036353980591898,5348.0,0.2035757851312988,0.0639002295208498,2472.0,34929.89146471024,35965.88739991188,34929.89146471024,964.7020807266236,4.014244794845581,0.0 +47500,0.4345858097076416,1.0648447275161743,,,,,,,,,,,,,, +48000,0.4999992251396179,1.1108893156051636,,,,,,,,,,,,,, +48500,0.4999993741512298,1.095747709274292,,,,,,,,,,,,,, +49000,0.4999992251396179,1.065677523612976,,,,,,,,,,,,,, +49500,0.4999994039535522,1.0886012315750122,,,,,,,,,,,,,, +49757,,,0.1135258299228843,0.0453112517003865,0.3629400810132861,0.1012890455269637,5348.0,0.1986226051882379,0.0615643978632218,2472.0,36675.30523777008,37760.6712975502,36675.30523777008,1009.7656865119934,4.100290775299072,0.0 +50000,0.4999993145465851,1.0563372373580933,,,,,,,,,,,,,, +50500,0.4999991953372955,1.0554252862930298,,,,,,,,,,,,,, +51000,0.4999991953372955,0.99881112575531,,,,,,,,,,,,,, +51500,0.4999991059303283,1.1088875532150269,,,,,,,,,,,,,, +52000,0.4999989569187164,1.0440136194229126,,,,,,,,,,,,,, +52211,,,0.1048411551697896,0.0417593332325696,0.354769562055769,0.0994930719837783,5348.0,0.195299028868798,0.0608738041557491,2472.0,38422.19172549248,39558.31389641762,38422.19172549248,1055.598946094513,4.455331563949585,0.0 +52500,0.4999991655349731,0.958897054195404,,,,,,,,,,,,,, +53000,0.4999991953372955,1.0781055688858032,,,,,,,,,,,,,, +53500,0.4999992549419403,1.0161510705947876,,,,,,,,,,,,,, +54000,0.4785270988941192,1.0628670454025269,,,,,,,,,,,,,, +54500,0.4999991953372955,1.0172115564346311,,,,,,,,,,,,,, +54520,,,0.0985952905152878,0.0396055103318722,0.3567766707322433,0.0972239656254526,5348.0,0.1944409593965739,0.0598582251741717,2472.0,40169.08051586151,41352.719628334045,40169.08051586151,1100.216621875763,4.762305021286011,0.0 +55000,0.4879449009895324,0.9992804527282716,,,,,,,,,,,,,, +55500,0.4999995827674866,1.061995029449463,,,,,,,,,,,,,, +56000,0.4999991357326507,0.993388295173645,,,,,,,,,,,,,, +56500,0.4999994933605194,1.0420708656311035,,,,,,,,,,,,,, +57000,0.4999992549419403,1.0364190340042114,,,,,,,,,,,,,, +57045,,,0.0929072879320841,0.0373113380692246,0.3505628822269941,0.0952059093323033,5348.0,0.1904134062076585,0.0587613998740682,2472.0,41914.625618219376,43147.09202218056,41914.625618219376,1144.8478198051453,4.847512245178223,0.0 +57500,0.4999995827674866,1.0029215812683103,,,,,,,,,,,,,, +58000,0.4999990165233612,0.987640380859375,,,,,,,,,,,,,, +58500,0.4999991953372955,1.024111032485962,,,,,,,,,,,,,, +59000,0.4999993741512298,0.9870431423187256,,,,,,,,,,,,,, +59464,,,0.0865763316076743,0.0347904476065035,0.3435994049188768,0.0935258050499686,5348.0,0.1856567611216852,0.0571770966628074,2472.0,43659.64862132072,44942.64661693573,43659.64862132072,1190.7182912826538,5.198246240615845,0.0 +59500,0.499999076128006,0.9468454122543336,,,,,,,,,,,,,, +60000,0.4999994337558746,1.029416799545288,,,,,,,,,,,,,, +60500,0.4999996721744537,0.9820741415023804,,,,,,,,,,,,,, +61000,0.4015413522720337,0.9195878505706788,,,,,,,,,,,,,, +61500,0.4999992549419403,0.9494996666908264,,,,,,,,,,,,,, +61769,,,0.0820577201747784,0.032782372120139,0.3425308420688291,0.0926760971370636,5348.0,0.1818494716392993,0.0564458797960717,2472.0,45405.07888031006,46735.14002013207,45405.07888031006,1234.8561778068542,5.529304265975952,0.0 +62000,0.4999991655349731,0.9475534558296204,,,,,,,,,,,,,, +62500,0.4999996423721313,1.0019614696502686,,,,,,,,,,,,,, +63000,0.4999991655349731,0.9123812913894652,,,,,,,,,,,,,, +63500,0.4999994337558746,0.9673970341682434,,,,,,,,,,,,,, +64000,0.4999993741512298,0.9447456002235411,,,,,,,,,,,,,, +64336,,,0.0778430698479672,0.031557014229266,0.3417285430951125,0.0907835658765026,5348.0,0.1816025429059939,0.0547803302662848,2472.0,47150.370564222336,48530.06073570252,47150.370564222336,1280.0275931358335,5.600770711898804,0.0 +64500,0.4881132841110229,1.0050970315933228,,,,,,,,,,,,,, +65000,0.4999991357326507,0.9941511750221252,,,,,,,,,,,,,, +65500,0.4999991953372955,0.9588475227355956,,,,,,,,,,,,,, +66000,0.4999991059303283,1.019245147705078,,,,,,,,,,,,,, +66500,0.4999995529651642,0.9542742371559144,,,,,,,,,,,,,, +66723,,,0.074476178560255,0.0303208602336277,0.3377734845916214,0.0900304156809733,5348.0,0.1792037742354345,0.0548006418459163,2472.0,48896.60098361969,50326.60763835907,48896.60098361969,1325.4059286117554,5.914076328277588,0.0 +67000,0.485116183757782,0.9166330099105836,,,,,,,,,,,,,, +67500,0.4999997317790985,0.9824973940849304,,,,,,,,,,,,,, +68000,0.4999991357326507,0.995658278465271,,,,,,,,,,,,,, +68500,0.4692767262458801,0.9169478416442872,,,,,,,,,,,,,, +69000,0.4999995231628418,0.8794509768486023,,,,,,,,,,,,,, +69036,,,0.0720835310724311,0.029090104290372,0.3336585107902602,0.0886399845507652,5348.0,0.1782793950416155,0.0540288018199175,2472.0,50642.02622580528,52118.62122583389,50642.02622580528,1369.0967593193054,6.2146546840667725,0.0 +69500,0.4999993145465851,0.9338986873626708,,,,,,,,,,,,,, +70000,0.4999990165233612,0.9477236866950988,,,,,,,,,,,,,, +70500,0.4999993741512298,1.0017024278640747,,,,,,,,,,,,,, +71000,0.499999076128006,0.8839278817176819,,,,,,,,,,,,,, +71500,0.4999996423721313,0.8767198324203491,,,,,,,,,,,,,, +71609,,,0.0706149747598593,0.0285934834711636,0.3356698135642706,0.0884179018008014,5348.0,0.178314276324279,0.0539069323421282,2472.0,52387.27734780312,53913.56525182724,52387.27734780312,1414.4519746303558,6.284870862960815,0.0 +72000,0.4999995231628418,0.9745815992355348,,,,,,,,,,,,,, +72500,0.4999990165233612,0.9662468433380128,,,,,,,,,,,,,, +73000,0.4999991357326507,0.9286428689956664,,,,,,,,,,,,,, +73500,0.4999991953372955,0.8560521602630615,,,,,,,,,,,,,, +73963,,,0.0698388642148075,0.028242610066288,0.3355244672367892,0.0882344421377878,5348.0,0.1773794629895477,0.053378831271708,2472.0,54135.1698012352,55711.57153177261,54135.1698012352,1459.8960573673248,6.639477014541626,0.0 +74000,0.4999994337558746,0.9183762669563292,,,,,,,,,,,,,, +74500,0.4999992549419403,0.9164686799049376,,,,,,,,,,,,,, +75000,0.499999463558197,0.9588059782981871,,,,,,,,,,,,,, +75500,0.4999993741512298,0.9965074062347412,,,,,,,,,,,,,, +76000,0.4999993145465851,0.9530206918716432,,,,,,,,,,,,,, +76310,,,0.0697052780220523,0.0282749983805842,0.3351290447778531,0.0884179018008014,5348.0,0.1771469001024259,0.0535210123291288,2472.0,55880.35697817802,57504.23585700989,55880.35697817802,1504.374971628189,6.962735891342163,0.0 +76500,0.4999992847442627,0.9502330422401428,,,,,,,,,,,,,, +77000,0.4999991655349731,0.9703825116157532,,,,,,,,,,,,,, +77500,0.499999076128006,0.9340662360191344,,,,,,,,,,,,,, +78000,0.4999989867210388,0.8855552077293396,,,,,,,,,,,,,, +78500,0.4674670398235321,0.932844340801239,,,,,,,,,,,,,, +78867,,,0.0696163614878576,0.0282048236996091,0.3350176927146122,0.0883406556269009,5348.0,0.1773022024606426,0.0536835049661812,2472.0,57625.57404828072,59299.41465139389,57625.57404828072,1549.9344420433044,7.044204473495483,0.0 +79000,0.4999996125698089,1.0100374221801758,,,,,,,,,,,,,, +79500,0.4999995231628418,0.9250512719154358,,,,,,,,,,,,,, +80000,0.4999991655349731,0.9213140606880188,,,,,,,,,,,,,, +80500,0.4999992251396179,0.9208353161811828,,,,,,,,,,,,,, +81000,0.4999992847442627,0.9014265537261964,,,,,,,,,,,,,, +81215,,,0.0693816707218595,0.027918726923326,0.3353849337813316,0.088688263409453,5348.0,0.1776324152969546,0.0538866207624967,2472.0,59370.740894556046,61093.5891122818,59370.740894556046,1595.9380114078522,7.333662748336792,0.0 +81500,0.499999076128006,0.8929963707923889,,,,,,,,,,,,,, +82000,0.4999991953372955,0.934037446975708,,,,,,,,,,,,,, +82500,0.4999993443489074,0.9541879892349244,,,,,,,,,,,,,, +83000,0.4579222798347473,0.941821277141571,,,,,,,,,,,,,, +83500,0.4999990165233612,0.9427101016044616,,,,,,,,,,,,,, +83625,,,0.0691009494647568,0.0279889016043011,0.3360513783325171,0.0882634094530005,5348.0,0.17802338054068,0.0544959681514431,2472.0,61116.29442381859,62885.84878635407,61116.29442381859,1639.9197208881378,7.427438497543335,0.0 +84000,0.4999995529651642,0.9953396320343018,,,,,,,,,,,,,, +84500,0.4999991059303283,0.9510433673858644,,,,,,,,,,,,,, +85000,0.4999992549419403,0.9351831078529358,,,,,,,,,,,,,, +85500,0.4999991655349731,0.935490608215332,,,,,,,,,,,,,, +86000,0.4999992549419403,0.915837287902832,,,,,,,,,,,,,, +86130,,,0.0690304768357234,0.0281022607043378,0.3365171929547795,0.0886399845507652,5348.0,0.1786100615362281,0.0544959681514431,2472.0,62861.324580192566,64681.55446791649,62861.324580192566,1685.8157725334167,7.788813591003418,0.0 +86500,0.4999997317790985,0.9486665725708008,,,,,,,,,,,,,, +87000,0.4999992549419403,0.9532247185707092,,,,,,,,,,,,,, +87500,0.4999993443489074,0.928145706653595,,,,,,,,,,,,,, +88000,0.4999995231628418,1.015143871307373,,,,,,,,,,,,,, +88461,,,0.0696058249446154,0.0281616392805475,0.3386531168374523,0.0890841500506928,5348.0,0.1772885927116265,0.0540897365588121,2472.0,64606.37130188942,66474.23734116554,64606.37130188942,1730.5017790794373,8.163705825805664,0.0 +88500,0.499999463558197,0.9513149857521056,,,,,,,,,,,,,, +89000,0.4999993145465851,0.948031485080719,,,,,,,,,,,,,, +89500,0.4999995827674866,1.0157896280288696,,,,,,,,,,,,,, +90000,0.4999992549419403,0.9126444458961488,,,,,,,,,,,,,, +90500,0.4999994933605194,0.9235316514968872,,,,,,,,,,,,,, +90928,,,,,,,,,,,66351.72258377075,,,,,0.0 diff --git a/logs/self_tuning/ademamix_golden/study_1/librispeech_conformer_pytorch/trial_1/meta_data_0.json b/logs/self_tuning/ademamix_golden/study_1/librispeech_conformer_pytorch/trial_1/meta_data_0.json new file mode 100644 index 00000000..eefc0e92 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/librispeech_conformer_pytorch/trial_1/meta_data_0.json @@ -0,0 +1,71 @@ +{ + "workload.attention_temperature": 1.0, + "workload.eval_batch_size": 256, + "workload.eval_num_workers": 0, + "workload.eval_period_time_sec": 1747, + "workload.max_allowed_runtime_sec": 43680, + "workload.num_eval_train_examples": 5376, + "workload.num_test_examples": 2472, + "workload.num_train_examples": 263840, + "workload.num_validation_examples": 5348, + "workload.requires_sync_before_eval": true, + "workload.step_hint": 76000, + "workload.target_metric_name": "wer", + "workload.test_target_value": 0.052981, + "workload.use_gelu": false, + "workload.use_post_layer_norm": true, + "workload.use_specaug": true, + "workload.validation_target_value": 0.085884, + "cpu.util.avg_percent_since_last": 14.8, + "cpu.freq.current": 2200.1959999999985, + "mem.total": 359053524992, + "mem.available": 349666488320, + "mem.used": 6231224320, + "mem.percent_used": 2.6, + "mem.read_bytes_since_boot": 1054014976, + "mem.write_bytes_since_boot": 30534641152, + "net.bytes_sent_since_boot": 31231683, + "net.bytes_recv_since_boot": 31235102, + "gpu.count": 4, + "gpu.0.compute.util": 0.08, + "gpu.0.mem.util": 0.0333251953125, + "gpu.0.mem.total": 40960.0, + "gpu.0.mem.used": 1365.0, + "gpu.0.mem.free": 38962.0, + "gpu.0.temp.current": 32.0, + "gpu.1.compute.util": 0.0, + "gpu.1.mem.util": 0.0333251953125, + "gpu.1.mem.total": 40960.0, + "gpu.1.mem.used": 1365.0, + "gpu.1.mem.free": 38962.0, + "gpu.1.temp.current": 30.0, + "gpu.2.compute.util": 0.0, + "gpu.2.mem.util": 0.0333251953125, + "gpu.2.mem.total": 40960.0, + "gpu.2.mem.used": 1365.0, + "gpu.2.mem.free": 38962.0, + "gpu.2.temp.current": 30.0, + "gpu.3.compute.util": 0.09, + "gpu.3.mem.util": 0.0333251953125, + "gpu.3.mem.total": 40960.0, + "gpu.3.mem.used": 1365.0, + "gpu.3.mem.free": 38962.0, + "gpu.3.temp.current": 31.0, + "gpu.avg.compute.util": 0.042499999999999996, + "gpu.avg.mem.util": 0.0333251953125, + "gpu.avg.mem.total": 40960.0, + "gpu.avg.mem.used": 1365.0, + "gpu.avg.mem.free": 38962.0, + "gpu.avg.temp.current": 30.75, + "os_platform": "Linux-6.1.0-44-cloud-amd64-x86_64-with-glibc2.31", + "python_version": "3.11.10", + "python_compiler": "GCC 9.4.0", + "git_branch": "main", + "git_commit_hash": "b21be29be0a1573fb4f78f849aea019cdb520862", + "cpu_model_name": "Intel(R) Xeon(R) CPU @ 2.20GHz", + "cpu_count": 24, + "gpu_model_name": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_driver": "550.90.12", + "rng_seed": -486301922 +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/librispeech_deepspeech_pytorch_09-14-2026-01-11-07.log b/logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/librispeech_deepspeech_pytorch_09-14-2026-01-11-07.log new file mode 100644 index 00000000..c6e677af --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/librispeech_deepspeech_pytorch_09-14-2026-01-11-07.log @@ -0,0 +1,1031 @@ +torchrun --redirects 1:0,2:0,3:0 --standalone --nnodes=1 --nproc_per_node=4 submission_runner.py --framework=pytorch --workload=librispeech_deepspeech --submission_path=submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py --data_dir=/data/librispeech --experiment_dir=/experiment_runs --experiment_name=submissions_a100/ademamix_golden/study_1 --overwrite=True --save_checkpoints=False --rng_seed=611108887 --librispeech_tokenizer_vocab_path=/data/librispeech/spm_model.vocab --torch_compile=true --tuning_ruleset=self 2>&1 | tee -a /logs/librispeech_deepspeech_pytorch_09-14-2026-01-11-07.log +W0914 01:11:33.156000 9 site-packages/torch/distributed/run.py:803] +W0914 01:11:33.156000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0914 01:11:33.156000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0914 01:11:33.156000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-14 01:11:48.222204: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-14 01:11:48.222204: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-14 01:11:48.222204: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-14 01:11:48.222239: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789348308.951430 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789348308.951401 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789348308.951426 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789348308.951462 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789348309.025353 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789348309.025344 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789348309.025352 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789348309.025368 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789348310.555456 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789348310.555455 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789348310.555452 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789348310.555498 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789348310.555498 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789348310.555500 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789348310.555500 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789348310.555501 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789348310.555503 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789348310.555504 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789348310.555504 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789348310.555506 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789348310.555490 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789348310.555527 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789348310.555530 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789348310.555532 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789348338.561466 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789348338.815245 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789348338.861468 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789348338.878873 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W914 01:12:28.549806920 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0914 01:12:30.924157 140367510193344 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/librispeech_deepspeech_pytorch. +I0914 01:12:30.924156 139972024980672 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/librispeech_deepspeech_pytorch. +I0914 01:12:30.924155 140236360443072 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/librispeech_deepspeech_pytorch. +I0914 01:12:30.924177 140543438132416 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/librispeech_deepspeech_pytorch. +I0914 01:12:31.142054 140367510193344 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/librispeech_deepspeech_pytorch/trial_1. +I0914 01:12:31.416460 140367510193344 submission_runner.py:242] Initializing dataset. +I0914 01:12:31.416654 140367510193344 input_pipeline.py:19] Loading split = train-clean-100 +I0914 01:12:31.472396 140367510193344 input_pipeline.py:19] Loading split = train-clean-360 +I0914 01:12:31.678717 140367510193344 input_pipeline.py:19] Loading split = train-other-500 +I0914 01:12:32.195022 140367510193344 submission_runner.py:251] Initializing model. +W0914 01:12:46.125149 140543438132416 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0914 01:12:46.126353 140236360443072 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0914 01:12:46.126463 140367510193344 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0914 01:12:46.127544 139972024980672 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +I0914 01:12:50.216230 140367510193344 submission_runner.py:294] Initializing optimizer. +I0914 01:12:50.216846 140367510193344 submission_runner.py:299] Initializing metrics bundle. +I0914 01:12:50.216994 140367510193344 submission_runner.py:321] Initializing checkpoint and logger. +I0914 01:12:50.219242 140367510193344 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_1/librispeech_deepspeech_pytorch/trial_1/meta_data_0.json. +I0914 01:12:50.219470 140367510193344 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0914 01:12:50.219392 140236360443072 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0914 01:12:50.219399 139972024980672 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0914 01:12:50.219430 140543438132416 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0914 01:12:50.219528 140367510193344 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0914 01:12:50.219546 140236360443072 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0914 01:12:50.219548 139972024980672 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0914 01:12:50.219591 140543438132416 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0914 01:12:50.652910 140367510193344 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_1/librispeech_deepspeech_pytorch/trial_1/flags_0.json. +I0914 01:12:50.667478 140367510193344 submission_runner.py:359] Starting training loop. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/algorithmic-efficiency/algoperf/workloads/librispeech_conformer/librispeech_pytorch/preprocessor.py:516: UserWarning: Specified kernel cache directory could not be created! This disables kernel caching. Specified directory is /root/.cache/torch/kernels. This warning will appear only once per process. (Triggered internally at /pytorch/aten/src/ATen/native/cuda/jit_utils.cpp:1487.) + spectrum = torch.abs(spectrum) +/algorithmic-efficiency/algoperf/workloads/librispeech_conformer/librispeech_pytorch/preprocessor.py:516: UserWarning: Specified kernel cache directory could not be created! This disables kernel caching. Specified directory is /root/.cache/torch/kernels. This warning will appear only once per process. (Triggered internally at /pytorch/aten/src/ATen/native/cuda/jit_utils.cpp:1487.) + spectrum = torch.abs(spectrum) +/algorithmic-efficiency/algoperf/workloads/librispeech_conformer/librispeech_pytorch/preprocessor.py:516: UserWarning: Specified kernel cache directory could not be created! This disables kernel caching. Specified directory is /root/.cache/torch/kernels. This warning will appear only once per process. (Triggered internally at /pytorch/aten/src/ATen/native/cuda/jit_utils.cpp:1487.) + spectrum = torch.abs(spectrum) +I0914 01:13:54.615807 140350540322560 logging_writer.py:48] [0] global_step=0, grad_norm=0.5, loss=33.564 +I0914 01:13:54.845523 140367510193344 submission.py:307] 0) loss = 33.564, grad_norm = 0.500 +I0914 01:13:55.478986 140367510193344 spec.py:333] Evaluating on the training split. +I0914 01:13:55.480007 140367510193344 input_pipeline.py:19] Loading split = train-clean-100 +I0914 01:13:55.505170 140367510193344 input_pipeline.py:19] Loading split = train-clean-360 +I0914 01:13:55.597079 140367510193344 input_pipeline.py:19] Loading split = train-other-500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 01:14:30.042919 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 01:14:30.044176 140367510193344 input_pipeline.py:19] Loading split = dev-clean +I0914 01:14:30.062897 140367510193344 input_pipeline.py:19] Loading split = dev-other +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 01:14:56.547351 140367510193344 spec.py:363] Evaluating on the test split. +I0914 01:14:56.548458 140367510193344 input_pipeline.py:19] Loading split = test-clean +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 01:15:08.395165 140367510193344 submission_runner.py:516] Time since start: 137.73s, Step: 1, {'train/ctc_loss': 30.424191357629397, 'train/wer': 1.9979008261786941, 'validation/ctc_loss': 29.203894163150494, 'validation/wer': 1.817293487181963, 'validation/num_examples': 5348, 'test/ctc_loss': 29.431779469643203, 'test/wer': 2.0699733918306826, 'test/num_examples': 2472, 'score': 64.17994022369385, 'total_duration': 137.72752594947815, 'accumulated_submission_time': 64.17994022369385, 'accumulated_eval_time': 72.91593790054321, 'accumulated_logging_time': 0} +I0914 01:15:08.476550 140347820734208 logging_writer.py:48] [1] accumulated_eval_time=72.9159, accumulated_logging_time=0, accumulated_submission_time=64.1799, global_step=1, preemption_count=0, score=64.1799, test/ctc_loss=29.4318, test/num_examples=2472, test/wer=2.06997, total_duration=137.728, train/ctc_loss=30.4242, train/wer=1.9979, validation/ctc_loss=29.2039, validation/num_examples=5348, validation/wer=1.81729 +I0914 01:15:10.023494 140347812341504 logging_writer.py:48] [1] global_step=1, grad_norm=0.5, loss=32.9492 +I0914 01:15:10.026724 140367510193344 submission.py:307] 1) loss = 32.949, grad_norm = 0.500 +I0914 01:15:10.813390 140347820734208 logging_writer.py:48] [2] global_step=2, grad_norm=0.5, loss=33.5158 +I0914 01:15:10.816715 140367510193344 submission.py:307] 2) loss = 33.516, grad_norm = 0.500 +I0914 01:15:11.368971 140347812341504 logging_writer.py:48] [3] global_step=3, grad_norm=0.5, loss=33.549 +I0914 01:15:11.372151 140367510193344 submission.py:307] 3) loss = 33.549, grad_norm = 0.500 +I0914 01:15:11.924476 140347820734208 logging_writer.py:48] [4] global_step=4, grad_norm=0.5, loss=33.0034 +I0914 01:15:11.928159 140367510193344 submission.py:307] 4) loss = 33.003, grad_norm = 0.500 +I0914 01:15:12.477031 140347812341504 logging_writer.py:48] [5] global_step=5, grad_norm=0.5, loss=33.1146 +I0914 01:15:12.480616 140367510193344 submission.py:307] 5) loss = 33.115, grad_norm = 0.500 +I0914 01:15:13.036382 140347820734208 logging_writer.py:48] [6] global_step=6, grad_norm=0.5, loss=33.2035 +I0914 01:15:13.039447 140367510193344 submission.py:307] 6) loss = 33.204, grad_norm = 0.500 +I0914 01:15:13.586102 140347812341504 logging_writer.py:48] [7] global_step=7, grad_norm=0.5, loss=31.9263 +I0914 01:15:13.589149 140367510193344 submission.py:307] 7) loss = 31.926, grad_norm = 0.500 +I0914 01:15:14.136313 140347820734208 logging_writer.py:48] [8] global_step=8, grad_norm=0.5, loss=32.1967 +I0914 01:15:14.139292 140367510193344 submission.py:307] 8) loss = 32.197, grad_norm = 0.500 +I0914 01:15:14.688286 140347812341504 logging_writer.py:48] [9] global_step=9, grad_norm=0.5, loss=31.6871 +I0914 01:15:14.691538 140367510193344 submission.py:307] 9) loss = 31.687, grad_norm = 0.500 +I0914 01:15:15.827447 140347820734208 logging_writer.py:48] [10] global_step=10, grad_norm=0.5, loss=31.477 +I0914 01:15:15.830601 140367510193344 submission.py:307] 10) loss = 31.477, grad_norm = 0.500 +I0914 01:15:16.668335 140347812341504 logging_writer.py:48] [11] global_step=11, grad_norm=0.5, loss=31.2849 +I0914 01:15:16.671520 140367510193344 submission.py:307] 11) loss = 31.285, grad_norm = 0.500 +I0914 01:15:17.660396 140347820734208 logging_writer.py:48] [12] global_step=12, grad_norm=0.5, loss=31.0215 +I0914 01:15:17.663484 140367510193344 submission.py:307] 12) loss = 31.021, grad_norm = 0.500 +I0914 01:15:20.349486 140347812341504 logging_writer.py:48] [13] global_step=13, grad_norm=0.5, loss=30.4822 +I0914 01:15:20.352599 140367510193344 submission.py:307] 13) loss = 30.482, grad_norm = 0.500 +I0914 01:15:21.223099 140347820734208 logging_writer.py:48] [14] global_step=14, grad_norm=0.5, loss=29.9339 +I0914 01:15:21.226194 140367510193344 submission.py:307] 14) loss = 29.934, grad_norm = 0.500 +I0914 01:15:22.220360 140347812341504 logging_writer.py:48] [15] global_step=15, grad_norm=0.5, loss=28.8034 +I0914 01:15:22.223482 140367510193344 submission.py:307] 15) loss = 28.803, grad_norm = 0.500 +I0914 01:15:23.192504 140347820734208 logging_writer.py:48] [16] global_step=16, grad_norm=0.5, loss=28.968 +I0914 01:15:23.195545 140367510193344 submission.py:307] 16) loss = 28.968, grad_norm = 0.500 +I0914 01:15:25.670973 140347812341504 logging_writer.py:48] [17] global_step=17, grad_norm=0.5, loss=28.6417 +I0914 01:15:25.674137 140367510193344 submission.py:307] 17) loss = 28.642, grad_norm = 0.500 +I0914 01:15:26.840403 140347820734208 logging_writer.py:48] [18] global_step=18, grad_norm=0.5, loss=27.9443 +I0914 01:15:26.843498 140367510193344 submission.py:307] 18) loss = 27.944, grad_norm = 0.500 +I0914 01:15:27.984046 140347812341504 logging_writer.py:48] [19] global_step=19, grad_norm=0.5, loss=27.4773 +I0914 01:15:27.987203 140367510193344 submission.py:307] 19) loss = 27.477, grad_norm = 0.500 +I0914 01:15:29.218903 140347820734208 logging_writer.py:48] [20] global_step=20, grad_norm=0.5, loss=26.7731 +I0914 01:15:29.221974 140367510193344 submission.py:307] 20) loss = 26.773, grad_norm = 0.500 +I0914 01:15:31.316745 140347812341504 logging_writer.py:48] [21] global_step=21, grad_norm=0.5, loss=26.7852 +I0914 01:15:31.319867 140367510193344 submission.py:307] 21) loss = 26.785, grad_norm = 0.500 +I0914 01:15:32.914737 140347820734208 logging_writer.py:48] [22] global_step=22, grad_norm=0.5, loss=26.6641 +I0914 01:15:32.917902 140367510193344 submission.py:307] 22) loss = 26.664, grad_norm = 0.500 +I0914 01:15:33.737715 140347812341504 logging_writer.py:48] [23] global_step=23, grad_norm=0.5, loss=25.7734 +I0914 01:15:33.740712 140367510193344 submission.py:307] 23) loss = 25.773, grad_norm = 0.500 +I0914 01:15:34.616261 140347820734208 logging_writer.py:48] [24] global_step=24, grad_norm=0.5, loss=25.3918 +I0914 01:15:34.619469 140367510193344 submission.py:307] 24) loss = 25.392, grad_norm = 0.500 +I0914 01:15:36.846212 140347812341504 logging_writer.py:48] [25] global_step=25, grad_norm=0.5, loss=25.0635 +I0914 01:15:36.849305 140367510193344 submission.py:307] 25) loss = 25.063, grad_norm = 0.500 +I0914 01:15:38.477385 140347820734208 logging_writer.py:48] [26] global_step=26, grad_norm=0.5, loss=24.3544 +I0914 01:15:38.480468 140367510193344 submission.py:307] 26) loss = 24.354, grad_norm = 0.500 +I0914 01:15:39.776504 140347812341504 logging_writer.py:48] [27] global_step=27, grad_norm=0.5, loss=24.1507 +I0914 01:15:39.779467 140367510193344 submission.py:307] 27) loss = 24.151, grad_norm = 0.500 +I0914 01:15:40.349715 140347820734208 logging_writer.py:48] [28] global_step=28, grad_norm=0.5, loss=23.2114 +I0914 01:15:40.352882 140367510193344 submission.py:307] 28) loss = 23.211, grad_norm = 0.500 +I0914 01:15:42.322992 140347812341504 logging_writer.py:48] [29] global_step=29, grad_norm=0.5, loss=22.484 +I0914 01:15:42.326074 140367510193344 submission.py:307] 29) loss = 22.484, grad_norm = 0.500 +I0914 01:15:44.178560 140347820734208 logging_writer.py:48] [30] global_step=30, grad_norm=0.5, loss=21.6145 +I0914 01:15:44.181658 140367510193344 submission.py:307] 30) loss = 21.615, grad_norm = 0.500 +I0914 01:15:45.304557 140347812341504 logging_writer.py:48] [31] global_step=31, grad_norm=0.5, loss=21.1035 +I0914 01:15:45.307900 140367510193344 submission.py:307] 31) loss = 21.103, grad_norm = 0.500 +I0914 01:15:45.853435 140347820734208 logging_writer.py:48] [32] global_step=32, grad_norm=0.5, loss=20.5497 +I0914 01:15:45.856453 140367510193344 submission.py:307] 32) loss = 20.550, grad_norm = 0.500 +I0914 01:15:47.722735 140347812341504 logging_writer.py:48] [33] global_step=33, grad_norm=0.5, loss=20.0577 +I0914 01:15:47.725760 140367510193344 submission.py:307] 33) loss = 20.058, grad_norm = 0.500 +I0914 01:15:50.198375 140347820734208 logging_writer.py:48] [34] global_step=34, grad_norm=0.5, loss=19.5763 +I0914 01:15:50.201422 140367510193344 submission.py:307] 34) loss = 19.576, grad_norm = 0.500 +I0914 01:15:50.864790 140347812341504 logging_writer.py:48] [35] global_step=35, grad_norm=0.5, loss=18.793 +I0914 01:15:50.867864 140367510193344 submission.py:307] 35) loss = 18.793, grad_norm = 0.500 +I0914 01:15:51.452874 140347820734208 logging_writer.py:48] [36] global_step=36, grad_norm=0.5, loss=17.8167 +I0914 01:15:51.455876 140367510193344 submission.py:307] 36) loss = 17.817, grad_norm = 0.500 +I0914 01:15:53.412421 140347812341504 logging_writer.py:48] [37] global_step=37, grad_norm=0.5, loss=16.8213 +I0914 01:15:53.415491 140367510193344 submission.py:307] 37) loss = 16.821, grad_norm = 0.500 +I0914 01:15:55.925027 140347820734208 logging_writer.py:48] [38] global_step=38, grad_norm=0.5, loss=16.6202 +I0914 01:15:55.928246 140367510193344 submission.py:307] 38) loss = 16.620, grad_norm = 0.500 +I0914 01:15:56.492069 140347812341504 logging_writer.py:48] [39] global_step=39, grad_norm=0.5, loss=16.0219 +I0914 01:15:56.495178 140367510193344 submission.py:307] 39) loss = 16.022, grad_norm = 0.500 +I0914 01:15:57.058189 140347820734208 logging_writer.py:48] [40] global_step=40, grad_norm=0.5, loss=15.0965 +I0914 01:15:57.061320 140367510193344 submission.py:307] 40) loss = 15.097, grad_norm = 0.500 +I0914 01:15:58.506327 140347812341504 logging_writer.py:48] [41] global_step=41, grad_norm=0.5, loss=14.4207 +I0914 01:15:58.509430 140367510193344 submission.py:307] 41) loss = 14.421, grad_norm = 0.500 +I0914 01:16:01.745688 140347820734208 logging_writer.py:48] [42] global_step=42, grad_norm=0.5, loss=13.4161 +I0914 01:16:01.748666 140367510193344 submission.py:307] 42) loss = 13.416, grad_norm = 0.500 +I0914 01:16:02.321455 140347812341504 logging_writer.py:48] [43] global_step=43, grad_norm=0.5, loss=13.2337 +I0914 01:16:02.324794 140367510193344 submission.py:307] 43) loss = 13.234, grad_norm = 0.500 +I0914 01:16:02.878126 140347820734208 logging_writer.py:48] [44] global_step=44, grad_norm=0.5, loss=12.3824 +I0914 01:16:02.881202 140367510193344 submission.py:307] 44) loss = 12.382, grad_norm = 0.500 +I0914 01:16:04.143689 140347812341504 logging_writer.py:48] [45] global_step=45, grad_norm=0.5, loss=11.7818 +I0914 01:16:04.146823 140367510193344 submission.py:307] 45) loss = 11.782, grad_norm = 0.500 +I0914 01:16:07.096979 140347820734208 logging_writer.py:48] [46] global_step=46, grad_norm=0.5, loss=11.0139 +I0914 01:16:07.099997 140367510193344 submission.py:307] 46) loss = 11.014, grad_norm = 0.500 +I0914 01:16:07.645421 140347812341504 logging_writer.py:48] [47] global_step=47, grad_norm=0.5, loss=10.6998 +I0914 01:16:07.648555 140367510193344 submission.py:307] 47) loss = 10.700, grad_norm = 0.500 +I0914 01:16:08.196866 140347820734208 logging_writer.py:48] [48] global_step=48, grad_norm=0.5, loss=10.7406 +I0914 01:16:08.199895 140367510193344 submission.py:307] 48) loss = 10.741, grad_norm = 0.500 +I0914 01:16:09.489574 140347812341504 logging_writer.py:48] [49] global_step=49, grad_norm=0.5, loss=10.2929 +I0914 01:16:09.492601 140367510193344 submission.py:307] 49) loss = 10.293, grad_norm = 0.500 +I0914 01:16:12.288501 140347820734208 logging_writer.py:48] [50] global_step=50, grad_norm=0.5, loss=9.86059 +I0914 01:16:12.291611 140367510193344 submission.py:307] 50) loss = 9.861, grad_norm = 0.500 +I0914 01:16:12.835295 140347812341504 logging_writer.py:48] [51] global_step=51, grad_norm=0.5, loss=9.26183 +I0914 01:16:12.838509 140367510193344 submission.py:307] 51) loss = 9.262, grad_norm = 0.500 +I0914 01:16:13.386771 140347820734208 logging_writer.py:48] [52] global_step=52, grad_norm=0.5, loss=9.01916 +I0914 01:16:13.390399 140367510193344 submission.py:307] 52) loss = 9.019, grad_norm = 0.500 +I0914 01:16:15.342009 140347812341504 logging_writer.py:48] [53] global_step=53, grad_norm=0.5, loss=8.87979 +I0914 01:16:15.345155 140367510193344 submission.py:307] 53) loss = 8.880, grad_norm = 0.500 +I0914 01:16:17.873898 140347820734208 logging_writer.py:48] [54] global_step=54, grad_norm=0.5, loss=8.6538 +I0914 01:16:17.877005 140367510193344 submission.py:307] 54) loss = 8.654, grad_norm = 0.500 +I0914 01:16:18.429029 140347812341504 logging_writer.py:48] [55] global_step=55, grad_norm=0.5, loss=8.58598 +I0914 01:16:18.432072 140367510193344 submission.py:307] 55) loss = 8.586, grad_norm = 0.500 +I0914 01:16:19.007438 140347820734208 logging_writer.py:48] [56] global_step=56, grad_norm=0.5, loss=8.23762 +I0914 01:16:19.010659 140367510193344 submission.py:307] 56) loss = 8.238, grad_norm = 0.500 +I0914 01:16:20.954285 140347812341504 logging_writer.py:48] [57] global_step=57, grad_norm=0.5, loss=8.13659 +I0914 01:16:20.957363 140367510193344 submission.py:307] 57) loss = 8.137, grad_norm = 0.500 +I0914 01:16:23.289526 140347820734208 logging_writer.py:48] [58] global_step=58, grad_norm=0.5, loss=7.82268 +I0914 01:16:23.292781 140367510193344 submission.py:307] 58) loss = 7.823, grad_norm = 0.500 +I0914 01:16:23.839562 140347812341504 logging_writer.py:48] [59] global_step=59, grad_norm=0.5, loss=7.802 +I0914 01:16:23.842623 140367510193344 submission.py:307] 59) loss = 7.802, grad_norm = 0.500 +I0914 01:16:24.393388 140347820734208 logging_writer.py:48] [60] global_step=60, grad_norm=0.5, loss=7.54337 +I0914 01:16:24.396443 140367510193344 submission.py:307] 60) loss = 7.543, grad_norm = 0.500 +I0914 01:16:26.457166 140347812341504 logging_writer.py:48] [61] global_step=61, grad_norm=0.5, loss=7.44107 +I0914 01:16:26.460223 140367510193344 submission.py:307] 61) loss = 7.441, grad_norm = 0.500 +I0914 01:16:28.488309 140347820734208 logging_writer.py:48] [62] global_step=62, grad_norm=0.5, loss=7.38207 +I0914 01:16:28.491348 140367510193344 submission.py:307] 62) loss = 7.382, grad_norm = 0.500 +I0914 01:16:29.041439 140347812341504 logging_writer.py:48] [63] global_step=63, grad_norm=0.5, loss=7.42942 +I0914 01:16:29.044590 140367510193344 submission.py:307] 63) loss = 7.429, grad_norm = 0.500 +I0914 01:16:29.614014 140347820734208 logging_writer.py:48] [64] global_step=64, grad_norm=0.5, loss=7.25326 +I0914 01:16:29.617103 140367510193344 submission.py:307] 64) loss = 7.253, grad_norm = 0.500 +I0914 01:16:31.963891 140347812341504 logging_writer.py:48] [65] global_step=65, grad_norm=0.5, loss=7.13664 +I0914 01:16:31.966941 140367510193344 submission.py:307] 65) loss = 7.137, grad_norm = 0.500 +I0914 01:16:34.291993 140347820734208 logging_writer.py:48] [66] global_step=66, grad_norm=0.5, loss=7.17767 +I0914 01:16:34.295140 140367510193344 submission.py:307] 66) loss = 7.178, grad_norm = 0.500 +I0914 01:16:34.838530 140347812341504 logging_writer.py:48] [67] global_step=67, grad_norm=0.5, loss=7.10601 +I0914 01:16:34.841636 140367510193344 submission.py:307] 67) loss = 7.106, grad_norm = 0.500 +I0914 01:16:35.397982 140347820734208 logging_writer.py:48] [68] global_step=68, grad_norm=0.5, loss=7.03311 +I0914 01:16:35.400987 140367510193344 submission.py:307] 68) loss = 7.033, grad_norm = 0.500 +I0914 01:16:37.542260 140347812341504 logging_writer.py:48] [69] global_step=69, grad_norm=0.5, loss=7.00554 +I0914 01:16:37.545309 140367510193344 submission.py:307] 69) loss = 7.006, grad_norm = 0.500 +I0914 01:16:39.885007 140347820734208 logging_writer.py:48] [70] global_step=70, grad_norm=0.5, loss=6.96517 +I0914 01:16:39.888087 140367510193344 submission.py:307] 70) loss = 6.965, grad_norm = 0.500 +I0914 01:16:40.431286 140347812341504 logging_writer.py:48] [71] global_step=71, grad_norm=0.5, loss=7.04484 +I0914 01:16:40.434375 140367510193344 submission.py:307] 71) loss = 7.045, grad_norm = 0.500 +I0914 01:16:40.980496 140347820734208 logging_writer.py:48] [72] global_step=72, grad_norm=0.5, loss=6.80661 +I0914 01:16:40.983623 140367510193344 submission.py:307] 72) loss = 6.807, grad_norm = 0.500 +I0914 01:16:43.133606 140347812341504 logging_writer.py:48] [73] global_step=73, grad_norm=0.5, loss=6.85859 +I0914 01:16:43.136629 140367510193344 submission.py:307] 73) loss = 6.859, grad_norm = 0.500 +I0914 01:16:45.623062 140347820734208 logging_writer.py:48] [74] global_step=74, grad_norm=0.5, loss=6.82518 +I0914 01:16:45.626369 140367510193344 submission.py:307] 74) loss = 6.825, grad_norm = 0.500 +I0914 01:16:46.190363 140347812341504 logging_writer.py:48] [75] global_step=75, grad_norm=0.5, loss=6.77251 +I0914 01:16:46.193485 140367510193344 submission.py:307] 75) loss = 6.773, grad_norm = 0.500 +I0914 01:16:46.747005 140347820734208 logging_writer.py:48] [76] global_step=76, grad_norm=0.5, loss=6.78786 +I0914 01:16:46.750228 140367510193344 submission.py:307] 76) loss = 6.788, grad_norm = 0.500 +I0914 01:16:48.284423 140347812341504 logging_writer.py:48] [77] global_step=77, grad_norm=0.5, loss=6.72257 +I0914 01:16:48.287485 140367510193344 submission.py:307] 77) loss = 6.723, grad_norm = 0.500 +I0914 01:16:51.335582 140347820734208 logging_writer.py:48] [78] global_step=78, grad_norm=0.5, loss=6.72423 +I0914 01:16:51.338682 140367510193344 submission.py:307] 78) loss = 6.724, grad_norm = 0.500 +I0914 01:16:51.885935 140347812341504 logging_writer.py:48] [79] global_step=79, grad_norm=0.5, loss=6.69594 +I0914 01:16:51.888958 140367510193344 submission.py:307] 79) loss = 6.696, grad_norm = 0.500 +I0914 01:16:52.438976 140347820734208 logging_writer.py:48] [80] global_step=80, grad_norm=0.5, loss=6.62187 +I0914 01:16:52.442046 140367510193344 submission.py:307] 80) loss = 6.622, grad_norm = 0.500 +I0914 01:16:53.619705 140347812341504 logging_writer.py:48] [81] global_step=81, grad_norm=0.5, loss=6.6357 +I0914 01:16:53.622833 140367510193344 submission.py:307] 81) loss = 6.636, grad_norm = 0.500 +I0914 01:16:56.956908 140347820734208 logging_writer.py:48] [82] global_step=82, grad_norm=0.5, loss=6.61925 +I0914 01:16:56.959973 140367510193344 submission.py:307] 82) loss = 6.619, grad_norm = 0.500 +I0914 01:16:57.501852 140347812341504 logging_writer.py:48] [83] global_step=83, grad_norm=0.5, loss=6.55001 +I0914 01:16:57.504951 140367510193344 submission.py:307] 83) loss = 6.550, grad_norm = 0.500 +I0914 01:16:58.071292 140347820734208 logging_writer.py:48] [84] global_step=84, grad_norm=0.5, loss=6.49666 +I0914 01:16:58.074453 140367510193344 submission.py:307] 84) loss = 6.497, grad_norm = 0.500 +I0914 01:16:59.029566 140347812341504 logging_writer.py:48] [85] global_step=85, grad_norm=0.5, loss=6.43787 +I0914 01:16:59.032735 140367510193344 submission.py:307] 85) loss = 6.438, grad_norm = 0.500 +I0914 01:17:02.351478 140347820734208 logging_writer.py:48] [86] global_step=86, grad_norm=0.5, loss=6.40718 +I0914 01:17:02.354617 140367510193344 submission.py:307] 86) loss = 6.407, grad_norm = 0.500 +I0914 01:17:02.912523 140347812341504 logging_writer.py:48] [87] global_step=87, grad_norm=0.5, loss=6.3454 +I0914 01:17:02.915570 140367510193344 submission.py:307] 87) loss = 6.345, grad_norm = 0.500 +I0914 01:17:03.466839 140347820734208 logging_writer.py:48] [88] global_step=88, grad_norm=0.5, loss=6.31585 +I0914 01:17:03.469977 140367510193344 submission.py:307] 88) loss = 6.316, grad_norm = 0.500 +I0914 01:17:04.638857 140347812341504 logging_writer.py:48] [89] global_step=89, grad_norm=0.5, loss=6.2926 +I0914 01:17:04.642112 140367510193344 submission.py:307] 89) loss = 6.293, grad_norm = 0.500 +I0914 01:17:07.882502 140347820734208 logging_writer.py:48] [90] global_step=90, grad_norm=0.5, loss=6.21976 +I0914 01:17:07.885675 140367510193344 submission.py:307] 90) loss = 6.220, grad_norm = 0.500 +I0914 01:17:08.444208 140347812341504 logging_writer.py:48] [91] global_step=91, grad_norm=0.5, loss=6.18248 +I0914 01:17:08.447351 140367510193344 submission.py:307] 91) loss = 6.182, grad_norm = 0.500 +I0914 01:17:09.002679 140347820734208 logging_writer.py:48] [92] global_step=92, grad_norm=0.5, loss=6.12457 +I0914 01:17:09.005835 140367510193344 submission.py:307] 92) loss = 6.125, grad_norm = 0.500 +I0914 01:17:10.076134 140347812341504 logging_writer.py:48] [93] global_step=93, grad_norm=0.5, loss=6.10385 +I0914 01:17:10.079272 140367510193344 submission.py:307] 93) loss = 6.104, grad_norm = 0.500 +I0914 01:17:12.874891 140347820734208 logging_writer.py:48] [94] global_step=94, grad_norm=0.5, loss=6.08601 +I0914 01:17:12.878169 140367510193344 submission.py:307] 94) loss = 6.086, grad_norm = 0.500 +I0914 01:17:13.428203 140347812341504 logging_writer.py:48] [95] global_step=95, grad_norm=0.5, loss=6.05647 +I0914 01:17:13.431322 140367510193344 submission.py:307] 95) loss = 6.056, grad_norm = 0.500 +I0914 01:17:13.983976 140347820734208 logging_writer.py:48] [96] global_step=96, grad_norm=0.499999, loss=6.0507 +I0914 01:17:13.987083 140367510193344 submission.py:307] 96) loss = 6.051, grad_norm = 0.500 +I0914 01:17:15.569726 140347812341504 logging_writer.py:48] [97] global_step=97, grad_norm=0.499999, loss=6.00343 +I0914 01:17:15.573441 140367510193344 submission.py:307] 97) loss = 6.003, grad_norm = 0.500 +I0914 01:17:18.432234 140347820734208 logging_writer.py:48] [98] global_step=98, grad_norm=0.5, loss=6.02465 +I0914 01:17:18.435425 140367510193344 submission.py:307] 98) loss = 6.025, grad_norm = 0.500 +I0914 01:17:18.991559 140347812341504 logging_writer.py:48] [99] global_step=99, grad_norm=0.5, loss=5.98735 +I0914 01:17:18.995120 140367510193344 submission.py:307] 99) loss = 5.987, grad_norm = 0.500 +I0914 01:17:19.540607 140347820734208 logging_writer.py:48] [100] global_step=100, grad_norm=0.5, loss=5.99951 +I0914 01:17:19.543656 140367510193344 submission.py:307] 100) loss = 6.000, grad_norm = 0.500 +I0914 01:26:26.756277 140347812341504 logging_writer.py:48] [500] global_step=500, grad_norm=0.5, loss=4.66848 +I0914 01:26:26.760203 140367510193344 submission.py:307] 500) loss = 4.668, grad_norm = 0.500 +I0914 01:37:59.238751 140347820734208 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.5, loss=3.09927 +I0914 01:37:59.242559 140367510193344 submission.py:307] 1000) loss = 3.099, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 01:39:16.347339 140367510193344 spec.py:333] Evaluating on the training split. +I0914 01:39:29.997584 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 01:39:49.961484 140367510193344 spec.py:363] Evaluating on the test split. +I0914 01:40:00.124588 140367510193344 submission_runner.py:516] Time since start: 1629.46s, Step: 1091, {'train/ctc_loss': 6.6299591170909435, 'train/wer': 0.9415415166935583, 'validation/ctc_loss': 6.5073090120805706, 'validation/wer': 0.896519094288611, 'validation/num_examples': 5348, 'test/ctc_loss': 6.470239208981225, 'test/wer': 0.8993154997664168, 'test/num_examples': 2472, 'score': 1510.1516211032867, 'total_duration': 1629.4568991661072, 'accumulated_submission_time': 1510.1516211032867, 'accumulated_eval_time': 116.69307732582092, 'accumulated_logging_time': 0.09182500839233398} +I0914 01:40:00.251374 140347820734208 logging_writer.py:48] [1091] accumulated_eval_time=116.693, accumulated_logging_time=0.091825, accumulated_submission_time=1510.15, global_step=1091, preemption_count=0, score=1510.15, test/ctc_loss=6.47024, test/num_examples=2472, test/wer=0.899315, total_duration=1629.46, train/ctc_loss=6.62996, train/wer=0.941542, validation/ctc_loss=6.50731, validation/num_examples=5348, validation/wer=0.896519 +I0914 01:45:12.518090 140347812341504 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.5, loss=2.65516 +I0914 01:45:12.522221 140367510193344 submission.py:307] 1500) loss = 2.655, grad_norm = 0.500 +I0914 01:55:58.376465 140347820734208 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.5, loss=2.33123 +I0914 01:55:58.380567 140367510193344 submission.py:307] 2000) loss = 2.331, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 02:01:34.820207 140347820734208 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.5, loss=2.27837 +I0914 02:01:34.827739 140367510193344 submission.py:307] 2500) loss = 2.278, grad_norm = 0.500 +I0914 02:04:09.038982 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 02:04:24.250022 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 02:04:44.514497 140367510193344 spec.py:363] Evaluating on the test split. +I0914 02:04:54.761846 140367510193344 submission_runner.py:516] Time since start: 3124.09s, Step: 2694, {'train/ctc_loss': 5.42037396233919, 'train/wer': 0.9354544522451447, 'validation/ctc_loss': 5.29343221004119, 'validation/wer': 0.8926085067349008, 'validation/num_examples': 5348, 'test/ctc_loss': 5.134036066843022, 'test/wer': 0.893242337456584, 'test/num_examples': 2472, 'score': 2956.7598633766174, 'total_duration': 3124.094222545624, 'accumulated_submission_time': 2956.7598633766174, 'accumulated_eval_time': 162.41574835777283, 'accumulated_logging_time': 0.22876238822937012} +I0914 02:04:54.940270 140347820734208 logging_writer.py:48] [2694] accumulated_eval_time=162.416, accumulated_logging_time=0.228762, accumulated_submission_time=2956.76, global_step=2694, preemption_count=0, score=2956.76, test/ctc_loss=5.13404, test/num_examples=2472, test/wer=0.893242, total_duration=3124.09, train/ctc_loss=5.42037, train/wer=0.935454, validation/ctc_loss=5.29343, validation/num_examples=5348, validation/wer=0.892609 +I0914 02:10:58.631908 140347812341504 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.5, loss=2.14062 +I0914 02:10:58.636255 140367510193344 submission.py:307] 3000) loss = 2.141, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 02:16:42.028352 140347820734208 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.5, loss=2.0026 +I0914 02:16:42.035819 140367510193344 submission.py:307] 3500) loss = 2.003, grad_norm = 0.500 +I0914 02:24:04.158747 140347812341504 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.5, loss=1.91275 +I0914 02:24:04.162911 140367510193344 submission.py:307] 4000) loss = 1.913, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 02:29:02.920372 140367510193344 spec.py:333] Evaluating on the training split. +I0914 02:29:18.595292 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 02:29:38.318853 140367510193344 spec.py:363] Evaluating on the test split. +I0914 02:29:48.753180 140367510193344 submission_runner.py:516] Time since start: 4618.09s, Step: 4397, {'train/ctc_loss': 0.7364908717024884, 'train/wer': 0.22863186750921424, 'validation/ctc_loss': 0.9592859575924252, 'validation/wer': 0.26521508231545404, 'validation/num_examples': 5348, 'test/ctc_loss': 0.6292866172817602, 'test/wer': 0.19675827189080494, 'test/num_examples': 2472, 'score': 4402.464949607849, 'total_duration': 4618.085517883301, 'accumulated_submission_time': 4402.464949607849, 'accumulated_eval_time': 208.24845600128174, 'accumulated_logging_time': 0.41771411895751953} +I0914 02:29:48.785576 140347820734208 logging_writer.py:48] [4397] accumulated_eval_time=208.248, accumulated_logging_time=0.417714, accumulated_submission_time=4402.46, global_step=4397, preemption_count=0, score=4402.46, test/ctc_loss=0.629287, test/num_examples=2472, test/wer=0.196758, total_duration=4618.09, train/ctc_loss=0.736491, train/wer=0.228632, validation/ctc_loss=0.959286, validation/num_examples=5348, validation/wer=0.265215 +I0914 02:30:47.852434 140347812341504 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.5, loss=1.83466 +I0914 02:30:47.856017 140367510193344 submission.py:307] 4500) loss = 1.835, grad_norm = 0.500 +I0914 02:38:12.074011 140347820734208 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.5, loss=1.78626 +I0914 02:38:12.078053 140367510193344 submission.py:307] 5000) loss = 1.786, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 02:44:18.890266 140347820734208 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.5, loss=1.76914 +I0914 02:44:18.897541 140367510193344 submission.py:307] 5500) loss = 1.769, grad_norm = 0.500 +I0914 02:51:00.769636 140347812341504 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.5, loss=1.66213 +I0914 02:51:00.773916 140367510193344 submission.py:307] 6000) loss = 1.662, grad_norm = 0.500 +I0914 02:53:57.526077 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 02:54:13.429793 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 02:54:33.300405 140367510193344 spec.py:363] Evaluating on the test split. +I0914 02:54:43.776950 140367510193344 submission_runner.py:516] Time since start: 6113.11s, Step: 6152, {'train/ctc_loss': 0.564567527578269, 'train/wer': 0.18293571347626422, 'validation/ctc_loss': 0.8027857833534258, 'validation/wer': 0.22731617824554629, 'validation/num_examples': 5348, 'test/ctc_loss': 0.49343879040583266, 'test/wer': 0.15859281376312637, 'test/num_examples': 2472, 'score': 5848.90784907341, 'total_duration': 6113.109282255173, 'accumulated_submission_time': 5848.90784907341, 'accumulated_eval_time': 254.4990930557251, 'accumulated_logging_time': 0.46029186248779297} +I0914 02:54:43.808921 140347820734208 logging_writer.py:48] [6152] accumulated_eval_time=254.499, accumulated_logging_time=0.460292, accumulated_submission_time=5848.91, global_step=6152, preemption_count=0, score=5848.91, test/ctc_loss=0.493439, test/num_examples=2472, test/wer=0.158593, total_duration=6113.11, train/ctc_loss=0.564568, train/wer=0.182936, validation/ctc_loss=0.802786, validation/num_examples=5348, validation/wer=0.227316 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 02:58:09.119921 140347820734208 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.5, loss=1.66411 +I0914 02:58:09.126978 140367510193344 submission.py:307] 6500) loss = 1.664, grad_norm = 0.500 +I0914 03:04:44.494704 140347812341504 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.5, loss=1.69131 +I0914 03:04:44.498869 140367510193344 submission.py:307] 7000) loss = 1.691, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 03:11:23.986433 140347820734208 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.5, loss=1.65192 +I0914 03:11:23.993794 140367510193344 submission.py:307] 7500) loss = 1.652, grad_norm = 0.500 +I0914 03:17:34.196652 140347812341504 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.5, loss=1.54514 +I0914 03:17:34.200795 140367510193344 submission.py:307] 8000) loss = 1.545, grad_norm = 0.500 +I0914 03:18:51.592248 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 03:19:08.375442 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 03:19:28.206618 140367510193344 spec.py:363] Evaluating on the test split. +I0914 03:19:38.593672 140367510193344 submission_runner.py:516] Time since start: 7607.93s, Step: 8069, {'train/ctc_loss': 0.4789078483051961, 'train/wer': 0.15715742724393048, 'validation/ctc_loss': 0.727484416597599, 'validation/wer': 0.2075218461835562, 'validation/num_examples': 5348, 'test/ctc_loss': 0.43258496515904254, 'test/wer': 0.1398046026039445, 'test/num_examples': 2472, 'score': 7294.322849035263, 'total_duration': 7607.926026582718, 'accumulated_submission_time': 7294.322849035263, 'accumulated_eval_time': 301.5002896785736, 'accumulated_logging_time': 0.502281665802002} +I0914 03:19:38.624888 140347820734208 logging_writer.py:48] [8069] accumulated_eval_time=301.5, accumulated_logging_time=0.502282, accumulated_submission_time=7294.32, global_step=8069, preemption_count=0, score=7294.32, test/ctc_loss=0.432585, test/num_examples=2472, test/wer=0.139805, total_duration=7607.93, train/ctc_loss=0.478908, train/wer=0.157157, validation/ctc_loss=0.727484, validation/num_examples=5348, validation/wer=0.207522 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 03:25:20.396697 140347820734208 logging_writer.py:48] [8500] global_step=8500, grad_norm=0.5, loss=1.51583 +I0914 03:25:20.404296 140367510193344 submission.py:307] 8500) loss = 1.516, grad_norm = 0.500 +I0914 03:31:16.145877 140347812341504 logging_writer.py:48] [9000] global_step=9000, grad_norm=0.5, loss=1.57761 +I0914 03:31:16.150352 140367510193344 submission.py:307] 9000) loss = 1.578, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 03:38:24.160569 140347820734208 logging_writer.py:48] [9500] global_step=9500, grad_norm=0.5, loss=1.55171 +I0914 03:38:24.167780 140367510193344 submission.py:307] 9500) loss = 1.552, grad_norm = 0.500 +I0914 03:43:46.977684 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 03:44:04.438347 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 03:44:24.029486 140367510193344 spec.py:363] Evaluating on the test split. +I0914 03:44:34.730237 140367510193344 submission_runner.py:516] Time since start: 9104.06s, Step: 9980, {'train/ctc_loss': 0.41243009587092727, 'train/wer': 0.13754175413224257, 'validation/ctc_loss': 0.659895283931083, 'validation/wer': 0.18920484719741226, 'validation/num_examples': 5348, 'test/ctc_loss': 0.37929803434899023, 'test/wer': 0.12272256413381269, 'test/num_examples': 2472, 'score': 8740.309307813644, 'total_duration': 9104.06256055832, 'accumulated_submission_time': 8740.309307813644, 'accumulated_eval_time': 349.25259923934937, 'accumulated_logging_time': 0.5435240268707275} +I0914 03:44:34.924870 140347820734208 logging_writer.py:48] [9980] accumulated_eval_time=349.253, accumulated_logging_time=0.543524, accumulated_submission_time=8740.31, global_step=9980, preemption_count=0, score=8740.31, test/ctc_loss=0.379298, test/num_examples=2472, test/wer=0.122723, total_duration=9104.06, train/ctc_loss=0.41243, train/wer=0.137542, validation/ctc_loss=0.659895, validation/num_examples=5348, validation/wer=0.189205 +I0914 03:44:52.181195 140347812341504 logging_writer.py:48] [10000] global_step=10000, grad_norm=0.5, loss=1.55636 +I0914 03:44:52.184899 140367510193344 submission.py:307] 10000) loss = 1.556, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 03:52:20.897331 140347820734208 logging_writer.py:48] [10500] global_step=10500, grad_norm=0.5, loss=1.55252 +I0914 03:52:20.904947 140367510193344 submission.py:307] 10500) loss = 1.553, grad_norm = 0.500 +I0914 03:57:45.211893 140347812341504 logging_writer.py:48] [11000] global_step=11000, grad_norm=0.5, loss=1.50845 +I0914 03:57:45.278979 140367510193344 submission.py:307] 11000) loss = 1.508, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 04:05:19.723007 140347820734208 logging_writer.py:48] [11500] global_step=11500, grad_norm=0.5, loss=1.63311 +I0914 04:05:19.732021 140367510193344 submission.py:307] 11500) loss = 1.633, grad_norm = 0.500 +I0914 04:08:43.477206 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 04:09:00.693689 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 04:09:20.685445 140367510193344 spec.py:363] Evaluating on the test split. +I0914 04:09:31.304190 140367510193344 submission_runner.py:516] Time since start: 10600.64s, Step: 11859, {'train/ctc_loss': 0.3743213390753654, 'train/wer': 0.12639832497800993, 'validation/ctc_loss': 0.624480716765873, 'validation/wer': 0.1779944962101096, 'validation/num_examples': 5348, 'test/ctc_loss': 0.34782813608942514, 'test/wer': 0.11248552799951252, 'test/num_examples': 2472, 'score': 10185.611597776413, 'total_duration': 10600.63650560379, 'accumulated_submission_time': 10185.611597776413, 'accumulated_eval_time': 397.0793631076813, 'accumulated_logging_time': 0.7489340305328369} +I0914 04:09:31.515865 140347820734208 logging_writer.py:48] [11859] accumulated_eval_time=397.079, accumulated_logging_time=0.748934, accumulated_submission_time=10185.6, global_step=11859, preemption_count=0, score=10185.6, test/ctc_loss=0.347828, test/num_examples=2472, test/wer=0.112486, total_duration=10600.6, train/ctc_loss=0.374321, train/wer=0.126398, validation/ctc_loss=0.624481, validation/num_examples=5348, validation/wer=0.177994 +I0914 04:11:29.866893 140347812341504 logging_writer.py:48] [12000] global_step=12000, grad_norm=0.5, loss=1.5476 +I0914 04:11:29.871514 140367510193344 submission.py:307] 12000) loss = 1.548, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 04:19:25.189220 140347820734208 logging_writer.py:48] [12500] global_step=12500, grad_norm=0.5, loss=1.44531 +I0914 04:19:25.196569 140367510193344 submission.py:307] 12500) loss = 1.445, grad_norm = 0.500 +I0914 04:24:27.795586 140347812341504 logging_writer.py:48] [13000] global_step=13000, grad_norm=0.5, loss=1.47073 +I0914 04:24:27.799681 140367510193344 submission.py:307] 13000) loss = 1.471, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 04:32:22.397188 140347820734208 logging_writer.py:48] [13500] global_step=13500, grad_norm=0.5, loss=1.37697 +I0914 04:32:22.406190 140367510193344 submission.py:307] 13500) loss = 1.377, grad_norm = 0.500 +I0914 04:33:39.566699 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 04:33:54.779821 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 04:34:14.291843 140367510193344 spec.py:363] Evaluating on the test split. +I0914 04:34:24.672649 140367510193344 submission_runner.py:516] Time since start: 12094.01s, Step: 13641, {'train/ctc_loss': 0.3424478947239814, 'train/wer': 0.11636654057233516, 'validation/ctc_loss': 0.5873597650755977, 'validation/wer': 0.1672959011248974, 'validation/num_examples': 5348, 'test/ctc_loss': 0.33118094312536295, 'test/wer': 0.10795604574167733, 'test/num_examples': 2472, 'score': 11631.358527183533, 'total_duration': 12094.005010128021, 'accumulated_submission_time': 11631.358527183533, 'accumulated_eval_time': 442.1850917339325, 'accumulated_logging_time': 0.9710085391998291} +I0914 04:34:24.739052 140347820734208 logging_writer.py:48] [13641] accumulated_eval_time=442.185, accumulated_logging_time=0.971009, accumulated_submission_time=11631.4, global_step=13641, preemption_count=0, score=11631.4, test/ctc_loss=0.331181, test/num_examples=2472, test/wer=0.107956, total_duration=12094, train/ctc_loss=0.342448, train/wer=0.116367, validation/ctc_loss=0.58736, validation/num_examples=5348, validation/wer=0.167296 +I0914 04:38:17.698074 140347812341504 logging_writer.py:48] [14000] global_step=14000, grad_norm=0.5, loss=1.41682 +I0914 04:38:17.702269 140367510193344 submission.py:307] 14000) loss = 1.417, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 04:46:28.055385 140347820734208 logging_writer.py:48] [14500] global_step=14500, grad_norm=0.5, loss=1.46324 +I0914 04:46:28.062833 140367510193344 submission.py:307] 14500) loss = 1.463, grad_norm = 0.500 +I0914 04:51:18.866571 140347812341504 logging_writer.py:48] [15000] global_step=15000, grad_norm=0.5, loss=1.43787 +I0914 04:51:18.871168 140367510193344 submission.py:307] 15000) loss = 1.438, grad_norm = 0.500 +I0914 04:58:33.062359 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 04:58:48.809632 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 04:59:09.147345 140367510193344 spec.py:363] Evaluating on the test split. +I0914 04:59:19.660404 140367510193344 submission_runner.py:516] Time since start: 13588.99s, Step: 15423, {'train/ctc_loss': 0.32713955400386246, 'train/wer': 0.11158538888679032, 'validation/ctc_loss': 0.5830462216037523, 'validation/wer': 0.16389706947327765, 'validation/num_examples': 5348, 'test/ctc_loss': 0.3212276296051358, 'test/wer': 0.10415778035057786, 'test/num_examples': 2472, 'score': 13077.367248296738, 'total_duration': 13588.992755651474, 'accumulated_submission_time': 13077.367248296738, 'accumulated_eval_time': 488.7829692363739, 'accumulated_logging_time': 1.0474531650543213} +I0914 04:59:19.720378 140347820734208 logging_writer.py:48] [15423] accumulated_eval_time=488.783, accumulated_logging_time=1.04745, accumulated_submission_time=13077.4, global_step=15423, preemption_count=0, score=13077.4, test/ctc_loss=0.321228, test/num_examples=2472, test/wer=0.104158, total_duration=13589, train/ctc_loss=0.32714, train/wer=0.111585, validation/ctc_loss=0.583046, validation/num_examples=5348, validation/wer=0.163897 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 05:00:17.299145 140347820734208 logging_writer.py:48] [15500] global_step=15500, grad_norm=0.5, loss=1.37777 +I0914 05:00:17.305870 140367510193344 submission.py:307] 15500) loss = 1.378, grad_norm = 0.500 +I0914 05:05:06.508023 140347812341504 logging_writer.py:48] [16000] global_step=16000, grad_norm=0.5, loss=1.42588 +I0914 05:05:06.512081 140367510193344 submission.py:307] 16000) loss = 1.426, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 05:13:25.138251 140347820734208 logging_writer.py:48] [16500] global_step=16500, grad_norm=0.5, loss=1.3671 +I0914 05:13:25.145623 140367510193344 submission.py:307] 16500) loss = 1.367, grad_norm = 0.500 +I0914 05:18:05.338862 140347812341504 logging_writer.py:48] [17000] global_step=17000, grad_norm=0.5, loss=1.33842 +I0914 05:18:05.343201 140367510193344 submission.py:307] 17000) loss = 1.338, grad_norm = 0.500 +I0914 05:23:27.481438 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 05:23:43.458703 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 05:24:03.944919 140367510193344 spec.py:363] Evaluating on the test split. +I0914 05:24:14.585870 140367510193344 submission_runner.py:516] Time since start: 15083.92s, Step: 17342, {'train/ctc_loss': 0.3086408433911581, 'train/wer': 0.1064912498448554, 'validation/ctc_loss': 0.5636925462596695, 'validation/wer': 0.16074928788683435, 'validation/num_examples': 5348, 'test/ctc_loss': 0.3085389581585909, 'test/wer': 0.10019702232242601, 'test/num_examples': 2472, 'score': 14522.733395814896, 'total_duration': 15083.918245077133, 'accumulated_submission_time': 14522.733395814896, 'accumulated_eval_time': 535.8872499465942, 'accumulated_logging_time': 1.1191730499267578} +I0914 05:24:14.870142 140347820734208 logging_writer.py:48] [17342] accumulated_eval_time=535.887, accumulated_logging_time=1.11917, accumulated_submission_time=14522.7, global_step=17342, preemption_count=0, score=14522.7, test/ctc_loss=0.308539, test/num_examples=2472, test/wer=0.100197, total_duration=15083.9, train/ctc_loss=0.308641, train/wer=0.106491, validation/ctc_loss=0.563693, validation/num_examples=5348, validation/wer=0.160749 +I0914 05:27:16.769597 140347812341504 logging_writer.py:48] [17500] global_step=17500, grad_norm=0.5, loss=1.38965 +I0914 05:27:16.773827 140367510193344 submission.py:307] 17500) loss = 1.390, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 05:32:04.858697 140347820734208 logging_writer.py:48] [18000] global_step=18000, grad_norm=0.5, loss=1.34737 +I0914 05:32:04.866001 140367510193344 submission.py:307] 18000) loss = 1.347, grad_norm = 0.500 +I0914 05:40:01.342240 140347812341504 logging_writer.py:48] [18500] global_step=18500, grad_norm=0.5, loss=1.50418 +I0914 05:40:01.346529 140367510193344 submission.py:307] 18500) loss = 1.504, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 05:44:59.112002 140347820734208 logging_writer.py:48] [19000] global_step=19000, grad_norm=0.5, loss=1.4048 +I0914 05:44:59.119277 140367510193344 submission.py:307] 19000) loss = 1.405, grad_norm = 0.500 +I0914 05:48:23.239828 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 05:48:40.733793 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 05:49:01.284548 140367510193344 spec.py:363] Evaluating on the test split. +I0914 05:49:11.683429 140367510193344 submission_runner.py:516] Time since start: 16581.02s, Step: 19260, {'train/ctc_loss': 0.2974399627065632, 'train/wer': 0.10311314492933502, 'validation/ctc_loss': 0.5551025930216495, 'validation/wer': 0.156558682952735, 'validation/num_examples': 5348, 'test/ctc_loss': 0.3041775124604813, 'test/wer': 0.09863303069079682, 'test/num_examples': 2472, 'score': 15967.952255010605, 'total_duration': 16581.015773534775, 'accumulated_submission_time': 15967.952255010605, 'accumulated_eval_time': 584.3306908607483, 'accumulated_logging_time': 1.4141464233398438} +I0914 05:49:11.816794 140347820734208 logging_writer.py:48] [19260] accumulated_eval_time=584.331, accumulated_logging_time=1.41415, accumulated_submission_time=15968, global_step=19260, preemption_count=0, score=15968, test/ctc_loss=0.304178, test/num_examples=2472, test/wer=0.098633, total_duration=16581, train/ctc_loss=0.29744, train/wer=0.103113, validation/ctc_loss=0.555103, validation/num_examples=5348, validation/wer=0.156559 +I0914 05:53:37.850569 140347812341504 logging_writer.py:48] [19500] global_step=19500, grad_norm=0.5, loss=1.36655 +I0914 05:53:37.854670 140367510193344 submission.py:307] 19500) loss = 1.367, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 05:58:59.620061 140347820734208 logging_writer.py:48] [20000] global_step=20000, grad_norm=0.5, loss=1.3697 +I0914 05:58:59.627506 140367510193344 submission.py:307] 20000) loss = 1.370, grad_norm = 0.500 +I0914 06:06:24.231401 140347812341504 logging_writer.py:48] [20500] global_step=20500, grad_norm=0.5, loss=1.45767 +I0914 06:06:24.235812 140367510193344 submission.py:307] 20500) loss = 1.458, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 06:11:57.343790 140347820734208 logging_writer.py:48] [21000] global_step=21000, grad_norm=0.5, loss=1.40277 +I0914 06:11:57.351487 140367510193344 submission.py:307] 21000) loss = 1.403, grad_norm = 0.500 +I0914 06:13:19.593070 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 06:13:35.906004 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 06:13:55.467765 140367510193344 spec.py:363] Evaluating on the test split. +I0914 06:14:05.953304 140367510193344 submission_runner.py:516] Time since start: 18075.29s, Step: 21132, {'train/ctc_loss': 0.28055885807597547, 'train/wer': 0.09743620184446687, 'validation/ctc_loss': 0.5357545098076151, 'validation/wer': 0.15392265726838217, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2917021864313827, 'test/wer': 0.09465196108301342, 'test/num_examples': 2472, 'score': 17413.341284513474, 'total_duration': 18075.28558087349, 'accumulated_submission_time': 17413.341284513474, 'accumulated_eval_time': 630.6906733512878, 'accumulated_logging_time': 1.559232234954834} +I0914 06:14:06.012763 140347820734208 logging_writer.py:48] [21132] accumulated_eval_time=630.691, accumulated_logging_time=1.55923, accumulated_submission_time=17413.3, global_step=21132, preemption_count=0, score=17413.3, test/ctc_loss=0.291702, test/num_examples=2472, test/wer=0.094652, total_duration=18075.3, train/ctc_loss=0.280559, train/wer=0.0974362, validation/ctc_loss=0.535755, validation/num_examples=5348, validation/wer=0.153923 +I0914 06:20:17.146582 140347812341504 logging_writer.py:48] [21500] global_step=21500, grad_norm=0.5, loss=1.34264 +I0914 06:20:17.151045 140367510193344 submission.py:307] 21500) loss = 1.343, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 06:26:15.277935 140347820734208 logging_writer.py:48] [22000] global_step=22000, grad_norm=0.5, loss=1.35761 +I0914 06:26:15.285353 140367510193344 submission.py:307] 22000) loss = 1.358, grad_norm = 0.500 +I0914 06:33:06.598905 140347812341504 logging_writer.py:48] [22500] global_step=22500, grad_norm=0.5, loss=1.34269 +I0914 06:33:06.603330 140367510193344 submission.py:307] 22500) loss = 1.343, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 06:38:14.011201 140367510193344 spec.py:333] Evaluating on the training split. +I0914 06:38:29.895202 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 06:38:50.122935 140367510193344 spec.py:363] Evaluating on the test split. +I0914 06:39:00.732868 140367510193344 submission_runner.py:516] Time since start: 19570.07s, Step: 22884, {'train/ctc_loss': 0.26893214886367084, 'train/wer': 0.09478660198261302, 'validation/ctc_loss': 0.5277967615876783, 'validation/wer': 0.14999275817119684, 'validation/num_examples': 5348, 'test/ctc_loss': 0.28784231039099295, 'test/wer': 0.09178802835496516, 'test/num_examples': 2472, 'score': 18859.01353430748, 'total_duration': 19570.065182209015, 'accumulated_submission_time': 18859.01353430748, 'accumulated_eval_time': 677.4123427867889, 'accumulated_logging_time': 1.6290030479431152} +I0914 06:39:00.825212 140347820734208 logging_writer.py:48] [22884] accumulated_eval_time=677.412, accumulated_logging_time=1.629, accumulated_submission_time=18859, global_step=22884, preemption_count=0, score=18859, test/ctc_loss=0.287842, test/num_examples=2472, test/wer=0.091788, total_duration=19570.1, train/ctc_loss=0.268932, train/wer=0.0947866, validation/ctc_loss=0.527797, validation/num_examples=5348, validation/wer=0.149993 +I0914 06:40:06.718025 140347812341504 logging_writer.py:48] [23000] global_step=23000, grad_norm=0.5, loss=1.37351 +I0914 06:40:06.721877 140367510193344 submission.py:307] 23000) loss = 1.374, grad_norm = 0.500 +I0914 06:46:59.439068 140347820734208 logging_writer.py:48] [23500] global_step=23500, grad_norm=0.5, loss=1.33489 +I0914 06:46:59.443179 140367510193344 submission.py:307] 23500) loss = 1.335, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 06:53:26.847877 140347820734208 logging_writer.py:48] [24000] global_step=24000, grad_norm=0.5, loss=1.32967 +I0914 06:53:26.855159 140367510193344 submission.py:307] 24000) loss = 1.330, grad_norm = 0.500 +I0914 06:59:43.245941 140347812341504 logging_writer.py:48] [24500] global_step=24500, grad_norm=0.5, loss=1.3507 +I0914 06:59:43.250210 140367510193344 submission.py:307] 24500) loss = 1.351, grad_norm = 0.500 +I0914 07:03:09.569687 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 07:03:25.960496 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 07:03:46.258136 140367510193344 spec.py:363] Evaluating on the test split. +I0914 07:03:56.887773 140367510193344 submission_runner.py:516] Time since start: 21066.22s, Step: 24679, {'train/ctc_loss': 0.2497926942865039, 'train/wer': 0.08689176573436008, 'validation/ctc_loss': 0.5030040530188868, 'validation/wer': 0.14299232366146863, 'validation/num_examples': 5348, 'test/ctc_loss': 0.27204854950319374, 'test/wer': 0.08713667661934069, 'test/num_examples': 2472, 'score': 20305.428166627884, 'total_duration': 21066.22006201744, 'accumulated_submission_time': 20305.428166627884, 'accumulated_eval_time': 724.7301826477051, 'accumulated_logging_time': 1.7316012382507324} +I0914 07:03:56.945368 140347820734208 logging_writer.py:48] [24679] accumulated_eval_time=724.73, accumulated_logging_time=1.7316, accumulated_submission_time=20305.4, global_step=24679, preemption_count=0, score=20305.4, test/ctc_loss=0.272049, test/num_examples=2472, test/wer=0.0871367, total_duration=21066.2, train/ctc_loss=0.249793, train/wer=0.0868918, validation/ctc_loss=0.503004, validation/num_examples=5348, validation/wer=0.142992 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 07:07:15.809614 140347820734208 logging_writer.py:48] [25000] global_step=25000, grad_norm=0.5, loss=1.33202 +I0914 07:07:15.816898 140367510193344 submission.py:307] 25000) loss = 1.332, grad_norm = 0.500 +I0914 07:13:25.664070 140347812341504 logging_writer.py:48] [25500] global_step=25500, grad_norm=0.5, loss=1.2105 +I0914 07:13:25.668401 140367510193344 submission.py:307] 25500) loss = 1.210, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 07:20:26.802157 140347820734208 logging_writer.py:48] [26000] global_step=26000, grad_norm=0.5, loss=1.25662 +I0914 07:20:26.809382 140367510193344 submission.py:307] 26000) loss = 1.257, grad_norm = 0.500 +I0914 07:26:17.088825 140347812341504 logging_writer.py:48] [26500] global_step=26500, grad_norm=0.5, loss=1.23034 +I0914 07:26:17.093028 140367510193344 submission.py:307] 26500) loss = 1.230, grad_norm = 0.500 +I0914 07:28:06.864875 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 07:28:23.901822 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 07:28:43.903407 140367510193344 spec.py:363] Evaluating on the test split. +I0914 07:28:54.451678 140367510193344 submission_runner.py:516] Time since start: 22563.78s, Step: 26599, {'train/ctc_loss': 0.2343841106029302, 'train/wer': 0.0825854914171312, 'validation/ctc_loss': 0.4907958444783504, 'validation/wer': 0.1400955921402018, 'validation/num_examples': 5348, 'test/ctc_loss': 0.26484289308987674, 'test/wer': 0.08538988077102756, 'test/num_examples': 2472, 'score': 21752.29531764984, 'total_duration': 22563.784057855606, 'accumulated_submission_time': 21752.29531764984, 'accumulated_eval_time': 772.3168234825134, 'accumulated_logging_time': 1.7998158931732178} +I0914 07:28:54.757494 140347820734208 logging_writer.py:48] [26599] accumulated_eval_time=772.317, accumulated_logging_time=1.79982, accumulated_submission_time=21752.3, global_step=26599, preemption_count=0, score=21752.3, test/ctc_loss=0.264843, test/num_examples=2472, test/wer=0.0853899, total_duration=22563.8, train/ctc_loss=0.234384, train/wer=0.0825855, validation/ctc_loss=0.490796, validation/num_examples=5348, validation/wer=0.140096 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 07:34:23.700980 140347820734208 logging_writer.py:48] [27000] global_step=27000, grad_norm=0.5, loss=1.30528 +I0914 07:34:23.717366 140367510193344 submission.py:307] 27000) loss = 1.305, grad_norm = 0.500 +I0914 07:39:59.784614 140347812341504 logging_writer.py:48] [27500] global_step=27500, grad_norm=0.5, loss=1.23653 +I0914 07:39:59.788824 140367510193344 submission.py:307] 27500) loss = 1.237, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 07:47:24.996914 140347820734208 logging_writer.py:48] [28000] global_step=28000, grad_norm=0.5, loss=1.21059 +I0914 07:47:25.004380 140367510193344 submission.py:307] 28000) loss = 1.211, grad_norm = 0.500 +I0914 07:52:51.071944 140347812341504 logging_writer.py:48] [28500] global_step=28500, grad_norm=0.5, loss=1.28114 +I0914 07:52:51.076063 140367510193344 submission.py:307] 28500) loss = 1.281, grad_norm = 0.500 +I0914 07:53:02.255570 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 07:53:19.022749 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 07:53:38.741233 140367510193344 spec.py:363] Evaluating on the test split. +I0914 07:53:49.322037 140367510193344 submission_runner.py:516] Time since start: 24058.65s, Step: 28513, {'train/ctc_loss': 0.22169122386678844, 'train/wer': 0.07876488713568001, 'validation/ctc_loss': 0.47626268334337957, 'validation/wer': 0.13412832520639212, 'validation/num_examples': 5348, 'test/ctc_loss': 0.25575561326537194, 'test/wer': 0.08187597749476977, 'test/num_examples': 2472, 'score': 23197.384291172028, 'total_duration': 24058.654323339462, 'accumulated_submission_time': 23197.384291172028, 'accumulated_eval_time': 819.3830456733704, 'accumulated_logging_time': 2.1159608364105225} +I0914 07:53:49.387367 140347820734208 logging_writer.py:48] [28513] accumulated_eval_time=819.383, accumulated_logging_time=2.11596, accumulated_submission_time=23197.4, global_step=28513, preemption_count=0, score=23197.4, test/ctc_loss=0.255756, test/num_examples=2472, test/wer=0.081876, total_duration=24058.7, train/ctc_loss=0.221691, train/wer=0.0787649, validation/ctc_loss=0.476263, validation/num_examples=5348, validation/wer=0.134128 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 08:01:21.985845 140347820734208 logging_writer.py:48] [29000] global_step=29000, grad_norm=0.5, loss=1.18954 +I0914 08:01:21.993497 140367510193344 submission.py:307] 29000) loss = 1.190, grad_norm = 0.500 +I0914 08:06:32.860721 140347812341504 logging_writer.py:48] [29500] global_step=29500, grad_norm=0.5, loss=1.21399 +I0914 08:06:32.864902 140367510193344 submission.py:307] 29500) loss = 1.214, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 08:14:19.039386 140347820734208 logging_writer.py:48] [30000] global_step=30000, grad_norm=0.5, loss=1.33545 +I0914 08:14:19.046665 140367510193344 submission.py:307] 30000) loss = 1.335, grad_norm = 0.500 +I0914 08:17:57.185555 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 08:18:13.802792 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 08:18:33.806434 140367510193344 spec.py:363] Evaluating on the test split. +I0914 08:18:44.464837 140367510193344 submission_runner.py:516] Time since start: 25553.80s, Step: 30386, {'train/ctc_loss': 0.2086407591312466, 'train/wer': 0.0738488271068636, 'validation/ctc_loss': 0.4649183117339763, 'validation/wer': 0.1311833148264375, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2501848405542293, 'test/wer': 0.07941827635935246, 'test/num_examples': 2472, 'score': 24642.763041496277, 'total_duration': 25553.797189712524, 'accumulated_submission_time': 24642.763041496277, 'accumulated_eval_time': 866.6622533798218, 'accumulated_logging_time': 2.1917943954467773} +I0914 08:18:44.536993 140347820734208 logging_writer.py:48] [30386] accumulated_eval_time=866.662, accumulated_logging_time=2.19179, accumulated_submission_time=24642.8, global_step=30386, preemption_count=0, score=24642.8, test/ctc_loss=0.250185, test/num_examples=2472, test/wer=0.0794183, total_duration=25553.8, train/ctc_loss=0.208641, train/wer=0.0738488, validation/ctc_loss=0.464918, validation/num_examples=5348, validation/wer=0.131183 +I0914 08:20:15.962463 140347812341504 logging_writer.py:48] [30500] global_step=30500, grad_norm=0.5, loss=1.1448 +I0914 08:20:15.966796 140367510193344 submission.py:307] 30500) loss = 1.145, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 08:28:23.869553 140347820734208 logging_writer.py:48] [31000] global_step=31000, grad_norm=0.5, loss=1.17084 +I0914 08:28:23.877188 140367510193344 submission.py:307] 31000) loss = 1.171, grad_norm = 0.500 +I0914 08:33:18.147118 140347812341504 logging_writer.py:48] [31500] global_step=31500, grad_norm=0.5, loss=1.23024 +I0914 08:33:18.151391 140367510193344 submission.py:307] 31500) loss = 1.230, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 08:41:23.000609 140347820734208 logging_writer.py:48] [32000] global_step=32000, grad_norm=0.5, loss=1.22407 +I0914 08:41:23.008176 140367510193344 submission.py:307] 32000) loss = 1.224, grad_norm = 0.500 +I0914 08:42:52.235657 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 08:43:07.699473 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 08:43:27.722776 140367510193344 spec.py:363] Evaluating on the test split. +I0914 08:43:38.388482 140367510193344 submission_runner.py:516] Time since start: 27047.72s, Step: 32163, {'train/ctc_loss': 0.1984771179571353, 'train/wer': 0.07116684924262456, 'validation/ctc_loss': 0.45680192361425054, 'validation/wer': 0.12875006034857336, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2421484349796761, 'test/wer': 0.0761278004590417, 'test/num_examples': 2472, 'score': 26088.131887435913, 'total_duration': 27047.72083067894, 'accumulated_submission_time': 26088.131887435913, 'accumulated_eval_time': 912.8148829936981, 'accumulated_logging_time': 2.274639129638672} +I0914 08:43:38.462419 140347820734208 logging_writer.py:48] [32163] accumulated_eval_time=912.815, accumulated_logging_time=2.27464, accumulated_submission_time=26088.1, global_step=32163, preemption_count=0, score=26088.1, test/ctc_loss=0.242148, test/num_examples=2472, test/wer=0.0761278, total_duration=27047.7, train/ctc_loss=0.198477, train/wer=0.0711668, validation/ctc_loss=0.456802, validation/num_examples=5348, validation/wer=0.12875 +I0914 08:47:08.687852 140347812341504 logging_writer.py:48] [32500] global_step=32500, grad_norm=0.5, loss=1.18797 +I0914 08:47:08.691954 140367510193344 submission.py:307] 32500) loss = 1.188, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 08:55:28.828948 140347820734208 logging_writer.py:48] [33000] global_step=33000, grad_norm=0.5, loss=1.24288 +I0914 08:55:28.836576 140367510193344 submission.py:307] 33000) loss = 1.243, grad_norm = 0.500 +I0914 09:00:12.039639 140347812341504 logging_writer.py:48] [33500] global_step=33500, grad_norm=0.5, loss=1.1458 +I0914 09:00:12.044062 140367510193344 submission.py:307] 33500) loss = 1.146, grad_norm = 0.500 +I0914 09:07:46.773953 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 09:08:02.162079 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 09:08:22.727636 140367510193344 spec.py:363] Evaluating on the test split. +I0914 09:08:33.343631 140367510193344 submission_runner.py:516] Time since start: 28542.68s, Step: 33954, {'train/ctc_loss': 0.1903303430895076, 'train/wer': 0.06793984167156833, 'validation/ctc_loss': 0.4504454867389994, 'validation/wer': 0.12671269251194903, 'validation/num_examples': 5348, 'test/ctc_loss': 0.23808272408703787, 'test/wer': 0.07480754778299108, 'test/num_examples': 2472, 'score': 27534.087806224823, 'total_duration': 28542.67593407631, 'accumulated_submission_time': 27534.087806224823, 'accumulated_eval_time': 959.3843200206757, 'accumulated_logging_time': 2.358907461166382} +I0914 09:08:33.407588 140347820734208 logging_writer.py:48] [33954] accumulated_eval_time=959.384, accumulated_logging_time=2.35891, accumulated_submission_time=27534.1, global_step=33954, preemption_count=0, score=27534.1, test/ctc_loss=0.238083, test/num_examples=2472, test/wer=0.0748075, total_duration=28542.7, train/ctc_loss=0.19033, train/wer=0.0679398, validation/ctc_loss=0.450445, validation/num_examples=5348, validation/wer=0.126713 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 09:09:18.913491 140347820734208 logging_writer.py:48] [34000] global_step=34000, grad_norm=0.5, loss=1.22436 +I0914 09:09:18.920696 140367510193344 submission.py:307] 34000) loss = 1.224, grad_norm = 0.500 +I0914 09:14:02.381222 140347812341504 logging_writer.py:48] [34500] global_step=34500, grad_norm=0.5, loss=1.13172 +I0914 09:14:02.386153 140367510193344 submission.py:307] 34500) loss = 1.132, grad_norm = 0.500 +I0914 09:22:15.088195 140347820734208 logging_writer.py:48] [35000] global_step=35000, grad_norm=0.5, loss=1.14354 +I0914 09:22:15.092450 140367510193344 submission.py:307] 35000) loss = 1.144, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 09:27:02.712731 140347820734208 logging_writer.py:48] [35500] global_step=35500, grad_norm=0.5, loss=1.15958 +I0914 09:27:02.720381 140367510193344 submission.py:307] 35500) loss = 1.160, grad_norm = 0.500 +I0914 09:32:43.393260 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 09:33:01.114384 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 09:33:21.122724 140367510193344 spec.py:363] Evaluating on the test split. +I0914 09:33:31.854501 140367510193344 submission_runner.py:516] Time since start: 30041.19s, Step: 35877, {'train/ctc_loss': 0.18574104801635316, 'train/wer': 0.06632633788604023, 'validation/ctc_loss': 0.44599287582253366, 'validation/wer': 0.12527398252305316, 'validation/num_examples': 5348, 'test/ctc_loss': 0.23579353909929673, 'test/wer': 0.07383259196067678, 'test/num_examples': 2472, 'score': 28980.851875543594, 'total_duration': 30041.186826705933, 'accumulated_submission_time': 28980.851875543594, 'accumulated_eval_time': 1007.8453965187073, 'accumulated_logging_time': 2.4333620071411133} +I0914 09:33:32.148358 140347820734208 logging_writer.py:48] [35877] accumulated_eval_time=1007.85, accumulated_logging_time=2.43336, accumulated_submission_time=28980.9, global_step=35877, preemption_count=0, score=28980.9, test/ctc_loss=0.235794, test/num_examples=2472, test/wer=0.0738326, total_duration=30041.2, train/ctc_loss=0.185741, train/wer=0.0663263, validation/ctc_loss=0.445993, validation/num_examples=5348, validation/wer=0.125274 +I0914 09:35:50.824177 140347812341504 logging_writer.py:48] [36000] global_step=36000, grad_norm=0.5, loss=1.21428 +I0914 09:35:50.828048 140367510193344 submission.py:307] 36000) loss = 1.214, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 09:41:02.041681 140347820734208 logging_writer.py:48] [36500] global_step=36500, grad_norm=0.5, loss=1.15466 +I0914 09:41:02.049241 140367510193344 submission.py:307] 36500) loss = 1.155, grad_norm = 0.500 +I0914 09:48:40.215459 140347812341504 logging_writer.py:48] [37000] global_step=37000, grad_norm=0.5, loss=1.17658 +I0914 09:48:40.219607 140367510193344 submission.py:307] 37000) loss = 1.177, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 09:54:01.180924 140347820734208 logging_writer.py:48] [37500] global_step=37500, grad_norm=0.5, loss=1.18022 +I0914 09:54:01.190212 140367510193344 submission.py:307] 37500) loss = 1.180, grad_norm = 0.500 +I0914 09:57:40.416916 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 09:57:57.018714 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 09:58:16.692176 140367510193344 spec.py:363] Evaluating on the test split. +I0914 09:58:27.190209 140367510193344 submission_runner.py:516] Time since start: 31536.52s, Step: 37792, {'train/ctc_loss': 0.1835047636340963, 'train/wer': 0.065743533843107, 'validation/ctc_loss': 0.44440971437361865, 'validation/wer': 0.12457876695794912, 'validation/num_examples': 5348, 'test/ctc_loss': 0.23440821786889476, 'test/wer': 0.07344667194767737, 'test/num_examples': 2472, 'score': 30426.722893953323, 'total_duration': 31536.52254796028, 'accumulated_submission_time': 30426.722893953323, 'accumulated_eval_time': 1054.6184527873993, 'accumulated_logging_time': 2.7380096912384033} +I0914 09:58:27.251928 140347820734208 logging_writer.py:48] [37792] accumulated_eval_time=1054.62, accumulated_logging_time=2.73801, accumulated_submission_time=30426.7, global_step=37792, preemption_count=0, score=30426.7, test/ctc_loss=0.234408, test/num_examples=2472, test/wer=0.0734467, total_duration=31536.5, train/ctc_loss=0.183505, train/wer=0.0657435, validation/ctc_loss=0.44441, validation/num_examples=5348, validation/wer=0.124579 +I0914 10:02:14.856916 140347812341504 logging_writer.py:48] [38000] global_step=38000, grad_norm=0.5, loss=1.20553 +I0914 10:02:14.860801 140367510193344 submission.py:307] 38000) loss = 1.206, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 10:07:57.478463 140347820734208 logging_writer.py:48] [38500] global_step=38500, grad_norm=0.5, loss=1.17663 +I0914 10:07:57.486229 140367510193344 submission.py:307] 38500) loss = 1.177, grad_norm = 0.500 +I0914 10:14:56.766824 140347812341504 logging_writer.py:48] [39000] global_step=39000, grad_norm=0.5, loss=1.20175 +I0914 10:14:56.771095 140367510193344 submission.py:307] 39000) loss = 1.202, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 10:20:55.387261 140347820734208 logging_writer.py:48] [39500] global_step=39500, grad_norm=0.5, loss=1.20963 +I0914 10:20:55.394892 140367510193344 submission.py:307] 39500) loss = 1.210, grad_norm = 0.500 +I0914 10:22:35.175570 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 10:22:51.200931 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 10:23:11.027219 140367510193344 spec.py:363] Evaluating on the test split. +I0914 10:23:22.047197 140367510193344 submission_runner.py:516] Time since start: 33031.38s, Step: 39667, {'train/ctc_loss': 0.18319023718322486, 'train/wer': 0.06554926582879592, 'validation/ctc_loss': 0.44429002316907773, 'validation/wer': 0.12429874957755999, 'validation/num_examples': 5348, 'test/ctc_loss': 0.23412289200109684, 'test/wer': 0.0732435561513619, 'test/num_examples': 2472, 'score': 31872.2521109581, 'total_duration': 33031.37944364548, 'accumulated_submission_time': 31872.2521109581, 'accumulated_eval_time': 1101.4898154735565, 'accumulated_logging_time': 2.8106141090393066} +I0914 10:23:22.128146 140347820734208 logging_writer.py:48] [39667] accumulated_eval_time=1101.49, accumulated_logging_time=2.81061, accumulated_submission_time=31872.3, global_step=39667, preemption_count=0, score=31872.3, test/ctc_loss=0.234123, test/num_examples=2472, test/wer=0.0732436, total_duration=33031.4, train/ctc_loss=0.18319, train/wer=0.0655493, validation/ctc_loss=0.44429, validation/num_examples=5348, validation/wer=0.124299 +I0914 10:28:43.822380 140347812341504 logging_writer.py:48] [40000] global_step=40000, grad_norm=0.5, loss=1.19453 +I0914 10:28:43.826618 140367510193344 submission.py:307] 40000) loss = 1.195, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 10:35:00.427460 140347820734208 logging_writer.py:48] [40500] global_step=40500, grad_norm=0.5, loss=1.12447 +I0914 10:35:00.436618 140367510193344 submission.py:307] 40500) loss = 1.124, grad_norm = 0.500 +I0914 10:41:24.223703 140347812341504 logging_writer.py:48] [41000] global_step=41000, grad_norm=0.5, loss=1.19324 +I0914 10:41:24.227813 140367510193344 submission.py:307] 41000) loss = 1.193, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 10:47:29.901931 140367510193344 spec.py:333] Evaluating on the training split. +I0914 10:47:44.806558 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 10:48:04.661913 140367510193344 spec.py:363] Evaluating on the test split. +I0914 10:48:15.252974 140367510193344 submission_runner.py:516] Time since start: 34524.59s, Step: 41448, {'train/ctc_loss': 0.181653533881753, 'train/wer': 0.06505819945928736, 'validation/ctc_loss': 0.4430559774179978, 'validation/wer': 0.12433737266451021, 'validation/num_examples': 5348, 'test/ctc_loss': 0.23319924370120654, 'test/wer': 0.0725732740235208, 'test/num_examples': 2472, 'score': 33317.68475842476, 'total_duration': 34524.58527302742, 'accumulated_submission_time': 33317.68475842476, 'accumulated_eval_time': 1146.8406782150269, 'accumulated_logging_time': 2.902109384536743} +I0914 10:48:15.319551 140347820734208 logging_writer.py:48] [41448] accumulated_eval_time=1146.84, accumulated_logging_time=2.90211, accumulated_submission_time=33317.7, global_step=41448, preemption_count=0, score=33317.7, test/ctc_loss=0.233199, test/num_examples=2472, test/wer=0.0725733, total_duration=34524.6, train/ctc_loss=0.181654, train/wer=0.0650582, validation/ctc_loss=0.443056, validation/num_examples=5348, validation/wer=0.124337 +I0914 10:48:45.959044 140347812341504 logging_writer.py:48] [41500] global_step=41500, grad_norm=0.5, loss=1.14339 +I0914 10:48:45.962891 140367510193344 submission.py:307] 41500) loss = 1.143, grad_norm = 0.500 +I0914 10:55:14.332050 140347820734208 logging_writer.py:48] [42000] global_step=42000, grad_norm=0.5, loss=1.23938 +I0914 10:55:14.336106 140367510193344 submission.py:307] 42000) loss = 1.239, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 11:02:02.151389 140347820734208 logging_writer.py:48] [42500] global_step=42500, grad_norm=0.5, loss=1.1315 +I0914 11:02:02.159483 140367510193344 submission.py:307] 42500) loss = 1.131, grad_norm = 0.500 +I0914 11:07:57.415790 140347812341504 logging_writer.py:48] [43000] global_step=43000, grad_norm=0.5, loss=1.12268 +I0914 11:07:57.419860 140367510193344 submission.py:307] 43000) loss = 1.123, grad_norm = 0.500 +I0914 11:12:23.060450 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 11:12:38.472529 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 11:12:58.065671 140367510193344 spec.py:363] Evaluating on the test split. +I0914 11:13:08.719454 140367510193344 submission_runner.py:516] Time since start: 36018.05s, Step: 43232, {'train/ctc_loss': 0.18153088304808546, 'train/wer': 0.0650528031255565, 'validation/ctc_loss': 0.4440422035111513, 'validation/wer': 0.12433737266451021, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2351312483869927, 'test/wer': 0.07389352669957143, 'test/num_examples': 2472, 'score': 34763.08761382103, 'total_duration': 36018.051757097244, 'accumulated_submission_time': 34763.08761382103, 'accumulated_eval_time': 1192.4994394779205, 'accumulated_logging_time': 2.9804887771606445} +I0914 11:13:08.778693 140347820734208 logging_writer.py:48] [43232] accumulated_eval_time=1192.5, accumulated_logging_time=2.98049, accumulated_submission_time=34763.1, global_step=43232, preemption_count=0, score=34763.1, test/ctc_loss=0.235131, test/num_examples=2472, test/wer=0.0738935, total_duration=36018.1, train/ctc_loss=0.181531, train/wer=0.0650528, validation/ctc_loss=0.444042, validation/num_examples=5348, validation/wer=0.124337 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 11:15:50.435521 140347820734208 logging_writer.py:48] [43500] global_step=43500, grad_norm=0.5, loss=1.16847 +I0914 11:15:50.442884 140367510193344 submission.py:307] 43500) loss = 1.168, grad_norm = 0.500 +I0914 11:21:43.934167 140347812341504 logging_writer.py:48] [44000] global_step=44000, grad_norm=0.5, loss=1.14206 +I0914 11:21:43.938315 140367510193344 submission.py:307] 44000) loss = 1.142, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 11:28:59.755441 140347820734208 logging_writer.py:48] [44500] global_step=44500, grad_norm=0.5, loss=1.13899 +I0914 11:28:59.762954 140367510193344 submission.py:307] 44500) loss = 1.139, grad_norm = 0.500 +I0914 11:34:30.512082 140347812341504 logging_writer.py:48] [45000] global_step=45000, grad_norm=0.5, loss=1.09776 +I0914 11:34:30.516281 140367510193344 submission.py:307] 45000) loss = 1.098, grad_norm = 0.500 +I0914 11:37:17.834513 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 11:37:35.467868 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 11:37:55.401204 140367510193344 spec.py:363] Evaluating on the test split. +I0914 11:38:05.872062 140367510193344 submission_runner.py:516] Time since start: 37515.20s, Step: 45155, {'train/ctc_loss': 0.1830968008407454, 'train/wer': 0.06548990615775642, 'validation/ctc_loss': 0.44619291710618847, 'validation/wer': 0.12551537681649205, 'validation/num_examples': 5348, 'test/ctc_loss': 0.23628722014323505, 'test/wer': 0.0737107224828875, 'test/num_examples': 2472, 'score': 36208.82337117195, 'total_duration': 37515.20438790321, 'accumulated_submission_time': 36208.82337117195, 'accumulated_eval_time': 1240.5367317199707, 'accumulated_logging_time': 3.049771547317505} +I0914 11:38:06.254002 140347820734208 logging_writer.py:48] [45155] accumulated_eval_time=1240.54, accumulated_logging_time=3.04977, accumulated_submission_time=36208.8, global_step=45155, preemption_count=0, score=36208.8, test/ctc_loss=0.236287, test/num_examples=2472, test/wer=0.0737107, total_duration=37515.2, train/ctc_loss=0.183097, train/wer=0.0654899, validation/ctc_loss=0.446193, validation/num_examples=5348, validation/wer=0.125515 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 11:42:55.110373 140347820734208 logging_writer.py:48] [45500] global_step=45500, grad_norm=0.5, loss=1.16591 +I0914 11:42:55.117949 140367510193344 submission.py:307] 45500) loss = 1.166, grad_norm = 0.500 +I0914 11:48:19.056077 140347812341504 logging_writer.py:48] [46000] global_step=46000, grad_norm=0.5, loss=1.17314 +I0914 11:48:19.060465 140367510193344 submission.py:307] 46000) loss = 1.173, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 11:55:57.393473 140347820734208 logging_writer.py:48] [46500] global_step=46500, grad_norm=0.5, loss=1.26381 +I0914 11:55:57.400923 140367510193344 submission.py:307] 46500) loss = 1.264, grad_norm = 0.500 +I0914 12:01:07.929450 140347812341504 logging_writer.py:48] [47000] global_step=47000, grad_norm=0.5, loss=1.20492 +I0914 12:01:07.934052 140367510193344 submission.py:307] 47000) loss = 1.205, grad_norm = 0.500 +I0914 12:02:14.646614 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 12:02:30.835984 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 12:02:50.783304 140367510193344 spec.py:363] Evaluating on the test split. +I0914 12:03:01.271553 140367510193344 submission_runner.py:516] Time since start: 39010.60s, Step: 47071, {'train/ctc_loss': 0.18817646226756904, 'train/wer': 0.06796682334022265, 'validation/ctc_loss': 0.4556509749535363, 'validation/wer': 0.12703133297928837, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2421711178947029, 'test/wer': 0.07606686572014705, 'test/num_examples': 2472, 'score': 37654.81112194061, 'total_duration': 39010.60394477844, 'accumulated_submission_time': 37654.81112194061, 'accumulated_eval_time': 1287.16175699234, 'accumulated_logging_time': 3.442328453063965} +I0914 12:03:01.336846 140347820734208 logging_writer.py:48] [47071] accumulated_eval_time=1287.16, accumulated_logging_time=3.44233, accumulated_submission_time=37654.8, global_step=47071, preemption_count=0, score=37654.8, test/ctc_loss=0.242171, test/num_examples=2472, test/wer=0.0760669, total_duration=39010.6, train/ctc_loss=0.188176, train/wer=0.0679668, validation/ctc_loss=0.455651, validation/num_examples=5348, validation/wer=0.127031 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 12:09:54.223288 140347820734208 logging_writer.py:48] [47500] global_step=47500, grad_norm=0.5, loss=1.17558 +I0914 12:09:54.232351 140367510193344 submission.py:307] 47500) loss = 1.176, grad_norm = 0.500 +I0914 12:14:53.305764 140347812341504 logging_writer.py:48] [48000] global_step=48000, grad_norm=0.5, loss=1.19566 +I0914 12:14:53.309794 140367510193344 submission.py:307] 48000) loss = 1.196, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 12:22:51.935823 140347820734208 logging_writer.py:48] [48500] global_step=48500, grad_norm=0.5, loss=1.26995 +I0914 12:22:51.943325 140367510193344 submission.py:307] 48500) loss = 1.270, grad_norm = 0.500 +I0914 12:27:09.433593 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 12:27:25.657991 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 12:27:45.452287 140367510193344 spec.py:363] Evaluating on the test split. +I0914 12:27:55.880939 140367510193344 submission_runner.py:516] Time since start: 40505.21s, Step: 48953, {'train/ctc_loss': 0.1970463188319217, 'train/wer': 0.07046532585761234, 'validation/ctc_loss': 0.47289401363459416, 'validation/wer': 0.13225510548930625, 'validation/num_examples': 5348, 'test/ctc_loss': 0.24946246532034325, 'test/wer': 0.08014949322608819, 'test/num_examples': 2472, 'score': 39100.520500421524, 'total_duration': 40505.21326804161, 'accumulated_submission_time': 39100.520500421524, 'accumulated_eval_time': 1333.6088554859161, 'accumulated_logging_time': 3.5180375576019287} +I0914 12:27:55.944867 140347820734208 logging_writer.py:48] [48953] accumulated_eval_time=1333.61, accumulated_logging_time=3.51804, accumulated_submission_time=39100.5, global_step=48953, preemption_count=0, score=39100.5, test/ctc_loss=0.249462, test/num_examples=2472, test/wer=0.0801495, total_duration=40505.2, train/ctc_loss=0.197046, train/wer=0.0704653, validation/ctc_loss=0.472894, validation/num_examples=5348, validation/wer=0.132255 +I0914 12:28:32.730197 140347812341504 logging_writer.py:48] [49000] global_step=49000, grad_norm=0.5, loss=1.29739 +I0914 12:28:32.734083 140367510193344 submission.py:307] 49000) loss = 1.297, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 12:36:53.884184 140347820734208 logging_writer.py:48] [49500] global_step=49500, grad_norm=0.5, loss=1.26141 +I0914 12:36:53.892212 140367510193344 submission.py:307] 49500) loss = 1.261, grad_norm = 0.500 +I0914 12:41:41.110287 140347812341504 logging_writer.py:48] [50000] global_step=50000, grad_norm=0.5, loss=1.2385 +I0914 12:41:41.114557 140367510193344 submission.py:307] 50000) loss = 1.239, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 12:49:50.949721 140347820734208 logging_writer.py:48] [50500] global_step=50500, grad_norm=0.5, loss=1.17139 +I0914 12:49:50.957280 140367510193344 submission.py:307] 50500) loss = 1.171, grad_norm = 0.500 +I0914 12:52:04.152124 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 12:52:20.149539 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 12:52:40.260826 140367510193344 spec.py:363] Evaluating on the test split. +I0914 12:52:50.628885 140367510193344 submission_runner.py:516] Time since start: 41999.96s, Step: 50743, {'train/ctc_loss': 0.2117411289058682, 'train/wer': 0.07659556097587299, 'validation/ctc_loss': 0.49530084482368897, 'validation/wer': 0.137372664510211, 'validation/num_examples': 5348, 'test/ctc_loss': 0.26321919962578233, 'test/wer': 0.08305404911339954, 'test/num_examples': 2472, 'score': 40546.38946771622, 'total_duration': 41999.961280584335, 'accumulated_submission_time': 40546.38946771622, 'accumulated_eval_time': 1380.0854613780975, 'accumulated_logging_time': 3.592219591140747} +I0914 12:52:50.697240 140347820734208 logging_writer.py:48] [50743] accumulated_eval_time=1380.09, accumulated_logging_time=3.59222, accumulated_submission_time=40546.4, global_step=50743, preemption_count=0, score=40546.4, test/ctc_loss=0.263219, test/num_examples=2472, test/wer=0.083054, total_duration=42000, train/ctc_loss=0.211741, train/wer=0.0765956, validation/ctc_loss=0.495301, validation/num_examples=5348, validation/wer=0.137373 +I0914 12:55:27.940757 140347812341504 logging_writer.py:48] [51000] global_step=51000, grad_norm=0.5, loss=1.25989 +I0914 12:55:27.944747 140367510193344 submission.py:307] 51000) loss = 1.260, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 13:04:00.145561 140347820734208 logging_writer.py:48] [51500] global_step=51500, grad_norm=0.5, loss=1.33069 +I0914 13:04:00.155110 140367510193344 submission.py:307] 51500) loss = 1.331, grad_norm = 0.500 +I0914 13:08:36.976768 140347812341504 logging_writer.py:48] [52000] global_step=52000, grad_norm=0.5, loss=1.18465 +I0914 13:08:36.981054 140367510193344 submission.py:307] 52000) loss = 1.185, grad_norm = 0.500 +I0914 13:16:46.244585 140347820734208 logging_writer.py:48] [52500] global_step=52500, grad_norm=0.5, loss=1.23957 +I0914 13:16:46.248793 140367510193344 submission.py:307] 52500) loss = 1.240, grad_norm = 0.500 +I0914 13:17:00.736625 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 13:17:16.771335 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 13:17:36.986912 140367510193344 spec.py:363] Evaluating on the test split. +I0914 13:17:47.520813 140367510193344 submission_runner.py:516] Time since start: 43496.85s, Step: 52512, {'train/ctc_loss': 0.22450426734321757, 'train/wer': 0.08121482264949194, 'validation/ctc_loss': 0.5048763864558218, 'validation/wer': 0.14074252884661806, 'validation/num_examples': 5348, 'test/ctc_loss': 0.27097809233660236, 'test/wer': 0.08575548920439542, 'test/num_examples': 2472, 'score': 41994.09754395485, 'total_duration': 43496.8531627655, 'accumulated_submission_time': 41994.09754395485, 'accumulated_eval_time': 1426.869449853897, 'accumulated_logging_time': 3.6710097789764404} +I0914 13:17:47.591827 140347820734208 logging_writer.py:48] [52512] accumulated_eval_time=1426.87, accumulated_logging_time=3.67101, accumulated_submission_time=41994.1, global_step=52512, preemption_count=0, score=41994.1, test/ctc_loss=0.270978, test/num_examples=2472, test/wer=0.0857555, total_duration=43496.9, train/ctc_loss=0.224504, train/wer=0.0812148, validation/ctc_loss=0.504876, validation/num_examples=5348, validation/wer=0.140743 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 13:22:28.216240 140347820734208 logging_writer.py:48] [53000] global_step=53000, grad_norm=0.5, loss=1.28761 +I0914 13:22:28.223824 140367510193344 submission.py:307] 53000) loss = 1.288, grad_norm = 0.500 +I0914 13:30:25.032767 140347812341504 logging_writer.py:48] [53500] global_step=53500, grad_norm=0.5, loss=1.30181 +I0914 13:30:25.036905 140367510193344 submission.py:307] 53500) loss = 1.302, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 13:35:32.715275 140347820734208 logging_writer.py:48] [54000] global_step=54000, grad_norm=0.5, loss=1.28754 +I0914 13:35:32.722729 140367510193344 submission.py:307] 54000) loss = 1.288, grad_norm = 0.500 +I0914 13:41:56.258532 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 13:42:12.771564 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 13:42:32.847074 140367510193344 spec.py:363] Evaluating on the test split. +I0914 13:42:44.219706 140367510193344 submission_runner.py:516] Time since start: 44993.55s, Step: 54430, {'train/ctc_loss': 0.2504103457689728, 'train/wer': 0.09013496230660889, 'validation/ctc_loss': 0.536555754125226, 'validation/wer': 0.14992516776903395, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2861785689802568, 'test/wer': 0.09123961570491337, 'test/num_examples': 2472, 'score': 43439.65407204628, 'total_duration': 44993.551918029785, 'accumulated_submission_time': 43439.65407204628, 'accumulated_eval_time': 1474.8302474021912, 'accumulated_logging_time': 3.7524282932281494} +I0914 13:42:44.534060 140347820734208 logging_writer.py:48] [54430] accumulated_eval_time=1474.83, accumulated_logging_time=3.75243, accumulated_submission_time=43439.7, global_step=54430, preemption_count=0, score=43439.7, test/ctc_loss=0.286179, test/num_examples=2472, test/wer=0.0912396, total_duration=44993.6, train/ctc_loss=0.25041, train/wer=0.090135, validation/ctc_loss=0.536556, validation/num_examples=5348, validation/wer=0.149925 +I0914 13:44:02.195290 140347812341504 logging_writer.py:48] [54500] global_step=54500, grad_norm=0.5, loss=1.32082 +I0914 13:44:02.199049 140367510193344 submission.py:307] 54500) loss = 1.321, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 13:49:34.422956 140347820734208 logging_writer.py:48] [55000] global_step=55000, grad_norm=0.5, loss=1.33239 +I0914 13:49:34.430582 140367510193344 submission.py:307] 55000) loss = 1.332, grad_norm = 0.500 +I0914 13:56:50.659475 140347812341504 logging_writer.py:48] [55500] global_step=55500, grad_norm=0.5, loss=1.31687 +I0914 13:56:50.663760 140367510193344 submission.py:307] 55500) loss = 1.317, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 14:02:38.625235 140347820734208 logging_writer.py:48] [56000] global_step=56000, grad_norm=0.5, loss=1.30066 +I0914 14:02:38.632602 140367510193344 submission.py:307] 56000) loss = 1.301, grad_norm = 0.500 +I0914 14:06:52.478057 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 14:07:08.766232 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 14:07:28.495050 140367510193344 spec.py:363] Evaluating on the test split. +I0914 14:07:39.083250 140367510193344 submission_runner.py:516] Time since start: 46488.42s, Step: 56344, {'train/ctc_loss': 0.26336414858729834, 'train/wer': 0.09437108428533654, 'validation/ctc_loss': 0.5419323418977295, 'validation/wer': 0.15158596050789358, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2942602647751468, 'test/wer': 0.09284423049580566, 'test/num_examples': 2472, 'score': 44885.20058083534, 'total_duration': 46488.41561603546, 'accumulated_submission_time': 44885.20058083534, 'accumulated_eval_time': 1521.4352769851685, 'accumulated_logging_time': 4.077461004257202} +I0914 14:07:39.141728 140347820734208 logging_writer.py:48] [56344] accumulated_eval_time=1521.44, accumulated_logging_time=4.07746, accumulated_submission_time=44885.2, global_step=56344, preemption_count=0, score=44885.2, test/ctc_loss=0.29426, test/num_examples=2472, test/wer=0.0928442, total_duration=46488.4, train/ctc_loss=0.263364, train/wer=0.0943711, validation/ctc_loss=0.541932, validation/num_examples=5348, validation/wer=0.151586 +I0914 14:10:28.636789 140347812341504 logging_writer.py:48] [56500] global_step=56500, grad_norm=0.5, loss=1.36711 +I0914 14:10:28.640991 140367510193344 submission.py:307] 56500) loss = 1.367, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 14:16:35.074933 140347820734208 logging_writer.py:48] [57000] global_step=57000, grad_norm=0.5, loss=1.37025 +I0914 14:16:35.084140 140367510193344 submission.py:307] 57000) loss = 1.370, grad_norm = 0.500 +I0914 14:23:12.954033 140347812341504 logging_writer.py:48] [57500] global_step=57500, grad_norm=0.5, loss=1.39964 +I0914 14:23:12.958542 140367510193344 submission.py:307] 57500) loss = 1.400, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 14:29:32.962239 140347820734208 logging_writer.py:48] [58000] global_step=58000, grad_norm=0.5, loss=1.38781 +I0914 14:29:32.969806 140367510193344 submission.py:307] 58000) loss = 1.388, grad_norm = 0.500 +I0914 14:31:47.521229 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 14:32:04.202185 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 14:32:23.833972 140367510193344 spec.py:363] Evaluating on the test split. +I0914 14:32:34.690645 140367510193344 submission_runner.py:516] Time since start: 47984.02s, Step: 58225, {'train/ctc_loss': 0.27624290742195007, 'train/wer': 0.09713940348926939, 'validation/ctc_loss': 0.5533760473961473, 'validation/wer': 0.15441510162699754, 'validation/num_examples': 5348, 'test/ctc_loss': 0.3016079162365314, 'test/wer': 0.09601283691832714, 'test/num_examples': 2472, 'score': 46331.20462226868, 'total_duration': 47984.02299404144, 'accumulated_submission_time': 46331.20462226868, 'accumulated_eval_time': 1568.6045529842377, 'accumulated_logging_time': 4.146382808685303} +I0914 14:32:34.764503 140347820734208 logging_writer.py:48] [58225] accumulated_eval_time=1568.6, accumulated_logging_time=4.14638, accumulated_submission_time=46331.2, global_step=58225, preemption_count=0, score=46331.2, test/ctc_loss=0.301608, test/num_examples=2472, test/wer=0.0960128, total_duration=47984, train/ctc_loss=0.276243, train/wer=0.0971394, validation/ctc_loss=0.553376, validation/num_examples=5348, validation/wer=0.154415 +I0914 14:36:56.265190 140347812341504 logging_writer.py:48] [58500] global_step=58500, grad_norm=0.5, loss=1.39829 +I0914 14:36:56.269532 140367510193344 submission.py:307] 58500) loss = 1.398, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 14:43:33.910578 140347820734208 logging_writer.py:48] [59000] global_step=59000, grad_norm=0.5, loss=1.3495 +I0914 14:43:33.918076 140367510193344 submission.py:307] 59000) loss = 1.349, grad_norm = 0.500 +I0914 14:49:34.027367 140347812341504 logging_writer.py:48] [59500] global_step=59500, grad_norm=0.5, loss=1.40031 +I0914 14:49:34.031606 140367510193344 submission.py:307] 59500) loss = 1.400, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 14:56:30.298471 140347820734208 logging_writer.py:48] [60000] global_step=60000, grad_norm=0.5, loss=1.35209 +I0914 14:56:30.305911 140367510193344 submission.py:307] 60000) loss = 1.352, grad_norm = 0.500 +I0914 14:56:42.578472 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 14:56:58.410999 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 14:57:18.218039 140367510193344 spec.py:363] Evaluating on the test split. +I0914 14:57:29.044404 140367510193344 submission_runner.py:516] Time since start: 49478.38s, Step: 60022, {'train/ctc_loss': 0.28328234906522054, 'train/wer': 0.09943824165861713, 'validation/ctc_loss': 0.5648793532279737, 'validation/wer': 0.15544826920291605, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2993794710545842, 'test/wer': 0.09627688745353725, 'test/num_examples': 2472, 'score': 47776.686915397644, 'total_duration': 49478.376770973206, 'accumulated_submission_time': 47776.686915397644, 'accumulated_eval_time': 1615.0702879428864, 'accumulated_logging_time': 4.230457782745361} +I0914 14:57:29.139951 140347820734208 logging_writer.py:48] [60022] accumulated_eval_time=1615.07, accumulated_logging_time=4.23046, accumulated_submission_time=47776.7, global_step=60022, preemption_count=0, score=47776.7, test/ctc_loss=0.299379, test/num_examples=2472, test/wer=0.0962769, total_duration=49478.4, train/ctc_loss=0.283282, train/wer=0.0994382, validation/ctc_loss=0.564879, validation/num_examples=5348, validation/wer=0.155448 +I0914 15:03:24.256819 140347812341504 logging_writer.py:48] [60500] global_step=60500, grad_norm=0.5, loss=1.31443 +I0914 15:03:24.261054 140367510193344 submission.py:307] 60500) loss = 1.314, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 15:10:34.130182 140347820734208 logging_writer.py:48] [61000] global_step=61000, grad_norm=0.5, loss=1.39163 +I0914 15:10:34.137689 140367510193344 submission.py:307] 61000) loss = 1.392, grad_norm = 0.500 +I0914 15:16:08.960340 140347812341504 logging_writer.py:48] [61500] global_step=61500, grad_norm=0.5, loss=1.32337 +I0914 15:16:08.964614 140367510193344 submission.py:307] 61500) loss = 1.323, grad_norm = 0.500 +I0914 15:21:37.955122 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 15:21:54.041249 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 15:22:13.699103 140367510193344 spec.py:363] Evaluating on the test split. +I0914 15:22:24.037896 140367510193344 submission_runner.py:516] Time since start: 50973.37s, Step: 61792, {'train/ctc_loss': 0.29928428051304173, 'train/wer': 0.10582750079595922, 'validation/ctc_loss': 0.5677088238710568, 'validation/wer': 0.15987061265871674, 'validation/num_examples': 5348, 'test/ctc_loss': 0.3120878010274856, 'test/wer': 0.10037982653910994, 'test/num_examples': 2472, 'score': 49223.1929461956, 'total_duration': 50973.3702442646, 'accumulated_submission_time': 49223.1929461956, 'accumulated_eval_time': 1661.152830839157, 'accumulated_logging_time': 4.336699724197388} +I0914 15:22:24.181173 140347820734208 logging_writer.py:48] [61792] accumulated_eval_time=1661.15, accumulated_logging_time=4.3367, accumulated_submission_time=49223.2, global_step=61792, preemption_count=0, score=49223.2, test/ctc_loss=0.312088, test/num_examples=2472, test/wer=0.10038, total_duration=50973.4, train/ctc_loss=0.299284, train/wer=0.105828, validation/ctc_loss=0.567709, validation/num_examples=5348, validation/wer=0.159871 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 15:24:21.811518 140347820734208 logging_writer.py:48] [62000] global_step=62000, grad_norm=0.5, loss=1.39635 +I0914 15:24:21.818320 140367510193344 submission.py:307] 62000) loss = 1.396, grad_norm = 0.500 +I0914 15:29:55.330036 140347812341504 logging_writer.py:48] [62500] global_step=62500, grad_norm=0.5, loss=1.40207 +I0914 15:29:55.334381 140367510193344 submission.py:307] 62500) loss = 1.402, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 15:37:31.142264 140347820734208 logging_writer.py:48] [63000] global_step=63000, grad_norm=0.5, loss=1.38986 +I0914 15:37:31.149474 140367510193344 submission.py:307] 63000) loss = 1.390, grad_norm = 0.500 +I0914 15:42:48.589861 140347812341504 logging_writer.py:48] [63500] global_step=63500, grad_norm=0.5, loss=1.45276 +I0914 15:42:48.594310 140367510193344 submission.py:307] 63500) loss = 1.453, grad_norm = 0.500 +I0914 15:46:33.052840 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 15:46:50.096708 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 15:47:09.900785 140367510193344 spec.py:363] Evaluating on the test split. +I0914 15:47:20.496639 140367510193344 submission_runner.py:516] Time since start: 52469.83s, Step: 63715, {'train/ctc_loss': 0.30567447320703334, 'train/wer': 0.10600557980907771, 'validation/ctc_loss': 0.5747124271649589, 'validation/wer': 0.15873123159368513, 'validation/num_examples': 5348, 'test/ctc_loss': 0.3137460481321376, 'test/wer': 0.10021733390205756, 'test/num_examples': 2472, 'score': 50669.54753446579, 'total_duration': 52469.82901930809, 'accumulated_submission_time': 50669.54753446579, 'accumulated_eval_time': 1708.5964970588684, 'accumulated_logging_time': 4.49121356010437} +I0914 15:47:20.827836 140347820734208 logging_writer.py:48] [63715] accumulated_eval_time=1708.6, accumulated_logging_time=4.49121, accumulated_submission_time=50669.5, global_step=63715, preemption_count=0, score=50669.5, test/ctc_loss=0.313746, test/num_examples=2472, test/wer=0.100217, total_duration=52469.8, train/ctc_loss=0.305674, train/wer=0.106006, validation/ctc_loss=0.574712, validation/num_examples=5348, validation/wer=0.158731 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 15:51:25.551989 140347820734208 logging_writer.py:48] [64000] global_step=64000, grad_norm=0.5, loss=1.43314 +I0914 15:51:25.559341 140367510193344 submission.py:307] 64000) loss = 1.433, grad_norm = 0.500 +I0914 15:56:36.503294 140347812341504 logging_writer.py:48] [64500] global_step=64500, grad_norm=0.5, loss=1.385 +I0914 15:56:36.507530 140367510193344 submission.py:307] 64500) loss = 1.385, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 16:04:26.535764 140347820734208 logging_writer.py:48] [65000] global_step=65000, grad_norm=0.5, loss=1.34963 +I0914 16:04:26.543112 140367510193344 submission.py:307] 65000) loss = 1.350, grad_norm = 0.500 +I0914 16:09:25.955772 140347812341504 logging_writer.py:48] [65500] global_step=65500, grad_norm=0.5, loss=1.3782 +I0914 16:09:25.960039 140367510193344 submission.py:307] 65500) loss = 1.378, grad_norm = 0.500 +I0914 16:11:28.403118 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 16:11:45.873354 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 16:12:05.833795 140367510193344 spec.py:363] Evaluating on the test split. +I0914 16:12:16.209470 140367510193344 submission_runner.py:516] Time since start: 53965.54s, Step: 65635, {'train/ctc_loss': 0.3088394334700591, 'train/wer': 0.10743021191402562, 'validation/ctc_loss': 0.5700153322470866, 'validation/wer': 0.16129966687587505, 'validation/num_examples': 5348, 'test/ctc_loss': 0.31395785615200983, 'test/wer': 0.10037982653910994, 'test/num_examples': 2472, 'score': 52114.71427941322, 'total_duration': 53965.541815280914, 'accumulated_submission_time': 52114.71427941322, 'accumulated_eval_time': 1756.4026470184326, 'accumulated_logging_time': 4.832993507385254} +I0914 16:12:16.291279 140347820734208 logging_writer.py:48] [65635] accumulated_eval_time=1756.4, accumulated_logging_time=4.83299, accumulated_submission_time=52114.7, global_step=65635, preemption_count=0, score=52114.7, test/ctc_loss=0.313958, test/num_examples=2472, test/wer=0.10038, total_duration=53965.5, train/ctc_loss=0.308839, train/wer=0.10743, validation/ctc_loss=0.570015, validation/num_examples=5348, validation/wer=0.1613 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 16:18:21.779588 140347820734208 logging_writer.py:48] [66000] global_step=66000, grad_norm=0.5, loss=1.40754 +I0914 16:18:21.787013 140367510193344 submission.py:307] 66000) loss = 1.408, grad_norm = 0.500 +I0914 16:23:14.807982 140347812341504 logging_writer.py:48] [66500] global_step=66500, grad_norm=0.5, loss=1.49373 +I0914 16:23:14.812619 140367510193344 submission.py:307] 66500) loss = 1.494, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 16:31:18.316760 140347820734208 logging_writer.py:48] [67000] global_step=67000, grad_norm=0.5, loss=1.45032 +I0914 16:31:18.324423 140367510193344 submission.py:307] 67000) loss = 1.450, grad_norm = 0.500 +I0914 16:36:02.410063 140347812341504 logging_writer.py:48] [67500] global_step=67500, grad_norm=0.5, loss=1.423 +I0914 16:36:02.421003 140367510193344 submission.py:307] 67500) loss = 1.423, grad_norm = 0.500 +I0914 16:36:25.131314 140367510193344 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 16:36:43.528850 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 16:37:03.304393 140367510193344 spec.py:363] Evaluating on the test split. +I0914 16:37:13.909568 140367510193344 submission_runner.py:516] Time since start: 55463.24s, Step: 67530, {'train/ctc_loss': 0.3181459933569486, 'train/wer': 0.11002584843857084, 'validation/ctc_loss': 0.5868998369060177, 'validation/wer': 0.16339496934292475, 'validation/num_examples': 5348, 'test/ctc_loss': 0.32258845328730884, 'test/wer': 0.10308126663010582, 'test/num_examples': 2472, 'score': 53560.22973561287, 'total_duration': 55463.241881370544, 'accumulated_submission_time': 53560.22973561287, 'accumulated_eval_time': 1805.1806757450104, 'accumulated_logging_time': 4.925171375274658} +I0914 16:37:13.968627 140347820734208 logging_writer.py:48] [67530] accumulated_eval_time=1805.18, accumulated_logging_time=4.92517, accumulated_submission_time=53560.2, global_step=67530, preemption_count=0, score=53560.2, test/ctc_loss=0.322588, test/num_examples=2472, test/wer=0.103081, total_duration=55463.2, train/ctc_loss=0.318146, train/wer=0.110026, validation/ctc_loss=0.5869, validation/num_examples=5348, validation/wer=0.163395 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 16:45:20.090748 140347820734208 logging_writer.py:48] [68000] global_step=68000, grad_norm=0.5, loss=1.40236 +I0914 16:45:20.098426 140367510193344 submission.py:307] 68000) loss = 1.402, grad_norm = 0.500 +I0914 16:50:00.354443 140347812341504 logging_writer.py:48] [68500] global_step=68500, grad_norm=0.5, loss=1.4217 +I0914 16:50:00.358839 140367510193344 submission.py:307] 68500) loss = 1.422, grad_norm = 0.500 +I0914 16:58:09.154176 140347820734208 logging_writer.py:48] [69000] global_step=69000, grad_norm=0.5, loss=1.48311 +I0914 16:58:09.158361 140367510193344 submission.py:307] 69000) loss = 1.483, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 17:01:21.883902 140367510193344 spec.py:333] Evaluating on the training split. +I0914 17:01:37.302637 140367510193344 spec.py:346] Evaluating on the validation split. +I0914 17:01:56.881455 140367510193344 spec.py:363] Evaluating on the test split. +I0914 17:02:07.566949 140367510193344 submission_runner.py:516] Time since start: 56956.90s, Step: 69341, {'train/ctc_loss': 0.3353715103757655, 'train/wer': 0.11489334146380949, 'validation/ctc_loss': 0.5987854187857896, 'validation/wer': 0.16774972239656255, 'validation/num_examples': 5348, 'test/ctc_loss': 0.32832823891863994, 'test/wer': 0.10590457619889099, 'test/num_examples': 2472, 'score': 55005.79437327385, 'total_duration': 56956.899238824844, 'accumulated_submission_time': 55005.79437327385, 'accumulated_eval_time': 1850.863557100296, 'accumulated_logging_time': 4.994636297225952} +I0914 17:02:07.699070 140347820734208 logging_writer.py:48] [69341] accumulated_eval_time=1850.86, accumulated_logging_time=4.99464, accumulated_submission_time=55005.8, global_step=69341, preemption_count=0, score=55005.8, test/ctc_loss=0.328328, test/num_examples=2472, test/wer=0.105905, total_duration=56956.9, train/ctc_loss=0.335372, train/wer=0.114893, validation/ctc_loss=0.598785, validation/num_examples=5348, validation/wer=0.16775 +I0914 17:03:46.038951 140347812341504 logging_writer.py:48] [69500] global_step=69500, grad_norm=0.5, loss=1.37181 +I0914 17:03:46.043001 140367510193344 submission.py:307] 69500) loss = 1.372, grad_norm = 0.500 +I0914 17:11:55.594964 140347820734208 logging_writer.py:48] [70000] global_step=70000, grad_norm=0.5, loss=1.55326 +I0914 17:11:55.599015 140367510193344 submission.py:307] 70000) loss = 1.553, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 17:16:56.190479 140347820734208 logging_writer.py:48] [70500] global_step=70500, grad_norm=0.5, loss=1.52765 +I0914 17:16:56.197936 140367510193344 submission.py:307] 70500) loss = 1.528, grad_norm = 0.500 +I0914 17:24:37.485588 140347812341504 logging_writer.py:48] [71000] global_step=71000, grad_norm=0.5, loss=1.49161 +I0914 17:24:37.489778 140367510193344 submission.py:307] 71000) loss = 1.492, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 17:26:15.148082 140347820734208 logging_writer.py:48] [71098] global_step=71098, preemption_count=0, score=56451.6 +I0914 17:26:15.646886 140367510193344 submission_runner.py:857] Final librispeech_deepspeech score: 56451.64101982117 diff --git a/logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..e259592f --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,40 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/ctc_loss,test/num_examples,test/wer,total_duration,train/ctc_loss,train/wer,validation/ctc_loss,validation/num_examples,validation/wer +72.91593790054321,0.0,64.17994022369385,1,0,64.17994022369385,29.431779469643203,2472,2.069973391830682,137.72752594947815,30.4241913576294,1.997900826178694,29.20389416315049,5348,1.817293487181963 +116.69307732582092,0.0918250083923339,1510.1516211032867,1091,0,1510.1516211032867,6.470239208981225,2472,0.8993154997664168,1629.4568991661072,6.629959117090944,0.9415415166935583,6.5073090120805706,5348,0.896519094288611 +162.41574835777283,0.2287623882293701,2956.759863376617,2694,0,2956.759863376617,5.134036066843022,2472,0.893242337456584,3124.094222545624,5.42037396233919,0.9354544522451448,5.29343221004119,5348,0.8926085067349008 +208.2484560012817,0.4177141189575195,4402.464949607849,4397,0,4402.464949607849,0.6292866172817602,2472,0.1967582718908049,4618.085517883301,0.7364908717024884,0.2286318675092142,0.9592859575924252,5348,0.265215082315454 +254.4990930557251,0.4602918624877929,5848.90784907341,6152,0,5848.90784907341,0.4934387904058326,2472,0.1585928137631263,6113.109282255173,0.564567527578269,0.1829357134762642,0.8027857833534258,5348,0.2273161782455462 +301.5002896785736,0.502281665802002,7294.322849035263,8069,0,7294.322849035263,0.4325849651590425,2472,0.1398046026039445,7607.926026582718,0.4789078483051961,0.1571574272439304,0.727484416597599,5348,0.2075218461835562 +349.25259923934937,0.5435240268707275,8740.309307813644,9980,0,8740.309307813644,0.3792980343489902,2472,0.1227225641338126,9104.06256055832,0.4124300958709272,0.1375417541322425,0.659895283931083,5348,0.1892048471974122 +397.0793631076813,0.7489340305328369,10185.611597776411,11859,0,10185.611597776411,0.3478281360894251,2472,0.1124855279995125,10600.63650560379,0.3743213390753654,0.1263983249780099,0.624480716765873,5348,0.1779944962101096 +442.1850917339325,0.9710085391998292,11631.358527183533,13641,0,11631.358527183533,0.3311809431253629,2472,0.1079560457416773,12094.00501012802,0.3424478947239814,0.1163665405723351,0.5873597650755977,5348,0.1672959011248974 +488.7829692363739,1.0474531650543213,13077.367248296738,15423,0,13077.367248296738,0.3212276296051358,2472,0.1041577803505778,13588.992755651474,0.3271395540038624,0.1115853888867903,0.5830462216037523,5348,0.1638970694732776 +535.8872499465942,1.1191730499267578,14522.733395814896,17342,0,14522.733395814896,0.3085389581585909,2472,0.100197022322426,15083.918245077131,0.3086408433911581,0.1064912498448554,0.5636925462596695,5348,0.1607492878868343 +584.3306908607483,1.4141464233398438,15967.952255010605,19260,0,15967.952255010605,0.3041775124604813,2472,0.0986330306907968,16581.015773534775,0.2974399627065632,0.103113144929335,0.5551025930216495,5348,0.156558682952735 +630.6906733512878,1.559232234954834,17413.341284513474,21132,0,17413.341284513474,0.2917021864313827,2472,0.0946519610830134,18075.28558087349,0.2805588580759754,0.0974362018444668,0.5357545098076151,5348,0.1539226572683821 +677.4123427867889,1.6290030479431152,18859.01353430748,22884,0,18859.01353430748,0.2878423103909929,2472,0.0917880283549651,19570.06518220901,0.2689321488636708,0.094786601982613,0.5277967615876783,5348,0.1499927581711968 +724.7301826477051,1.7316012382507324,20305.428166627884,24679,0,20305.428166627884,0.2720485495031937,2472,0.0871366766193406,21066.22006201744,0.2497926942865039,0.08689176573436,0.5030040530188868,5348,0.1429923236614686 +772.3168234825134,1.7998158931732178,21752.29531764984,26599,0,21752.29531764984,0.2648428930898767,2472,0.0853898807710275,22563.78405785561,0.2343841106029302,0.0825854914171312,0.4907958444783504,5348,0.1400955921402018 +819.3830456733704,2.1159608364105225,23197.384291172028,28513,0,23197.384291172028,0.2557556132653719,2472,0.0818759774947697,24058.654323339462,0.2216912238667884,0.07876488713568,0.4762626833433795,5348,0.1341283252063921 +866.6622533798218,2.1917943954467773,24642.76304149628,30386,0,24642.76304149628,0.2501848405542293,2472,0.0794182763593524,25553.797189712524,0.2086407591312466,0.0738488271068636,0.4649183117339763,5348,0.1311833148264375 +912.814882993698,2.274639129638672,26088.13188743592,32163,0,26088.13188743592,0.2421484349796761,2472,0.0761278004590417,27047.72083067894,0.1984771179571353,0.0711668492426245,0.4568019236142505,5348,0.1287500603485733 +959.3843200206755,2.358907461166382,27534.087806224823,33954,0,27534.087806224823,0.2380827240870378,2472,0.074807547782991,28542.67593407631,0.1903303430895076,0.0679398416715683,0.4504454867389994,5348,0.126712692511949 +1007.8453965187072,2.4333620071411133,28980.851875543594,35877,0,28980.851875543594,0.2357935390992967,2472,0.0738325919606767,30041.186826705933,0.1857410480163531,0.0663263378860402,0.4459928758225336,5348,0.1252739825230531 +1054.6184527873993,2.738009691238404,30426.722893953323,37792,0,30426.722893953323,0.2344082178688947,2472,0.0734466719476773,31536.52254796028,0.1835047636340963,0.065743533843107,0.4444097143736186,5348,0.1245787669579491 +1101.4898154735563,2.810614109039306,31872.2521109581,39667,0,31872.2521109581,0.2341228920010968,2472,0.0732435561513619,33031.37944364548,0.1831902371832248,0.0655492658287959,0.4442900231690777,5348,0.1242987495775599 +1146.8406782150269,2.902109384536743,33317.68475842476,41448,0,33317.68475842476,0.2331992437012065,2472,0.0725732740235208,34524.58527302742,0.181653533881753,0.0650581994592873,0.4430559774179978,5348,0.1243373726645102 +1192.4994394779203,2.9804887771606445,34763.08761382103,43232,0,34763.08761382103,0.2351312483869927,2472,0.0738935266995714,36018.051757097244,0.1815308830480854,0.0650528031255565,0.4440422035111513,5348,0.1243373726645102 +1240.5367317199707,3.049771547317505,36208.82337117195,45155,0,36208.82337117195,0.236287220143235,2472,0.0737107224828875,37515.20438790321,0.1830968008407454,0.0654899061577564,0.4461929171061884,5348,0.125515376816492 +1287.16175699234,3.442328453063965,37654.81112194061,47071,0,37654.81112194061,0.2421711178947029,2472,0.076066865720147,39010.60394477844,0.188176462267569,0.0679668233402226,0.4556509749535363,5348,0.1270313329792883 +1333.608855485916,3.5180375576019287,39100.52050042152,48953,0,39100.52050042152,0.2494624653203432,2472,0.0801494932260881,40505.21326804161,0.1970463188319217,0.0704653258576123,0.4728940136345941,5348,0.1322551054893062 +1380.0854613780975,3.592219591140747,40546.38946771622,50743,0,40546.38946771622,0.2632191996257823,2472,0.0830540491133995,41999.961280584335,0.2117411289058682,0.0765955609758729,0.4953008448236889,5348,0.137372664510211 +1426.869449853897,3.67100977897644,41994.09754395485,52512,0,41994.09754395485,0.2709780923366023,2472,0.0857554892043954,43496.8531627655,0.2245042673432175,0.0812148226494919,0.5048763864558218,5348,0.140742528846618 +1474.8302474021912,3.752428293228149,43439.65407204628,54430,0,43439.65407204628,0.2861785689802568,2472,0.0912396157049133,44993.551918029785,0.2504103457689728,0.0901349623066088,0.536555754125226,5348,0.1499251677690339 +1521.4352769851685,4.077461004257202,44885.20058083534,56344,0,44885.20058083534,0.2942602647751468,2472,0.0928442304958056,46488.41561603546,0.2633641485872983,0.0943710842853365,0.5419323418977295,5348,0.1515859605078935 +1568.6045529842377,4.146382808685303,46331.20462226868,58225,0,46331.20462226868,0.3016079162365314,2472,0.0960128369183271,47984.02299404144,0.27624290742195,0.0971394034892693,0.5533760473961473,5348,0.1544151016269975 +1615.0702879428864,4.230457782745361,47776.68691539765,60022,0,47776.68691539765,0.2993794710545842,2472,0.0962768874535372,49478.376770973206,0.2832823490652205,0.0994382416586171,0.5648793532279737,5348,0.155448269202916 +1661.152830839157,4.336699724197388,49223.1929461956,61792,0,49223.1929461956,0.3120878010274856,2472,0.1003798265391099,50973.3702442646,0.2992842805130417,0.1058275007959592,0.5677088238710568,5348,0.1598706126587167 +1708.5964970588684,4.49121356010437,50669.54753446579,63715,0,50669.54753446579,0.3137460481321376,2472,0.1002173339020575,52469.82901930809,0.3056744732070333,0.1060055798090777,0.5747124271649589,5348,0.1587312315936851 +1756.4026470184326,4.832993507385254,52114.71427941322,65635,0,52114.71427941322,0.3139578561520098,2472,0.1003798265391099,53965.541815280914,0.3088394334700591,0.1074302119140256,0.5700153322470866,5348,0.161299666875875 +1805.1806757450104,4.925171375274658,53560.22973561287,67530,0,53560.22973561287,0.3225884532873088,2472,0.1030812666301058,55463.241881370544,0.3181459933569486,0.1100258484385708,0.5868998369060177,5348,0.1633949693429247 +1850.863557100296,4.994636297225952,55005.79437327385,69341,0,55005.79437327385,0.32832823891863994,2472,0.10590457619889099,56956.899238824844,0.3353715103757655,0.11489334146380949,0.5987854187857896,5348,0.16774972239656255 diff --git a/logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/trial_1/events.out.tfevents.1789348370.925fa6ef3e4d.38.0.v2 b/logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/trial_1/events.out.tfevents.1789348370.925fa6ef3e4d.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..9cabba368b53d3f31291b9f3d48b08621d1063db GIT binary patch literal 93254 zcmbT9c|Z@(otlIy#P&m#x=aO09v*2$_-N8q3ScH*&O^zhb_F zWAMz(o&EmS9^ba@o!oBEzqQ@vyUNQgc9_4=!p7EN<^Kcz`=1VnfEFp%y4rsOngpoM zl3P+gyypL0UB0_~4b z>hk?h&LjQ~O6tO$i*kxtOpcCxfBDu#j*at53+pxWSK6(xW)@vaHNX%iYJT1-{Xu^N z-xfVEk?Z`13Dl5RmsfDIc66fVd#uw3lzrqKyGNY`JwkWp>3BSGr92ZOhO-dP5EY`1YRbWw#YQwbnx}Qf1_5;>o7%b_%9|( zO-qhs5=Z`xTl>u>P)?ye6E{F!TV92X zTVcC+@lu<`kXT&{)T_dz-VXSim=Qm(P|m3flPKHtykJrAA1&7w$3(A|6c@ZIGP>&&h`156rVJ^LeAWG%l{XXYKdh@^>KPO z?m0dJI@+VNOtpkf(c|H`@h~4|!Ayk2^YP05Qvy(Q?HN`>Ax$@SEhSE4-(Q75_8FvzcveGEpQh06&!PA|^h^Vws29sr z^Hcn{?6|E?_O*Wm6nP0ugfN8xQZ#=&sRdAkPpGV+Xh2hJtKR^3)Sot{dO*=(CQ~gT z#g>V-xE8zSF%iNPE=bY&l=2Ipm>-p0Y&>wOoT8+ zEK-bJbLcBjbUkKWL(zz)XtMMVs#h`T%r2nVc800ur=X6D&QU)`0)_uYCPJ7Z2Pr0u zSQZ5ox=-C|C}e30$3!VSE>g@)ZGfWQOQu>vip835J^{s)JSIYzLJDWIm!6{*P>fR# zs-cjhDTZ(Uh1He&vgQDVhZa*UA;p*kGjpJb*JUDvDU@&=O#{CA0L9&=-)ku3X^M=L zN~}IUM_&;r8mchW5>jZ(j=lmEOH`Q%VG30o$8D7Xhk@czj>ZL1k7`U)tTd{?YD;wm zRiK#uj;WTALO*0=eV~Z?!bAvDXyG`_4jXm@ia%z{YABk}6!kUBvHFIO@h_mzv}UR$ zq|j8I22prA)Pi6!y+~zCf|bo{12q zu*7jVR6H*Oit9T2YABl06!t3%vASpS0}Y@^)Mu(Cq)__Qpbt>YGGrozDeQ0@o0~Y{ zTFhUbTtlHmQ*4WQi`713D++*OsUuU(PeHl6EGwxeP@G%OLY5Vg-J`MT0#nsbs#paj)f{8j2P)#pWlk zu)6EU{;@zI)q$zzr=WI}`h?^gKvCJ1i4dmn!Eq>iMcf1mv-XpML_4Y_O);f!239|4 zQIZW5v%50Y{1nt$C^(+X1Bwt$CPJ9P56AJRfvG=Gd^Jz2p-`qN`WSO4ihsX$0*bSX znQDHDU)&vaSf;57P&Bh+B7`Zjgm3>g}Hfx>hgQ_W979T$;q3J-x|!ZapA zn1VBo-l@+f07b)q6EzfVX^Q*lcaUQE zyfX9@P#k{nrG}y%O%bVc8>?mO?;HXYYo0RI5>iNQyc`A;cV03P!W7SO99H$V<^#ox z^{p<6c2s+sqAK({R(sCbau_I9?_jF=DX6;#t3rAn1d3IAnFwKu9301>2iieEVI1#S zL(zez==b_6Rv*{cd=V(pA2QYa6x7juaOcpgK%tevLho^b_=zB?O-iq@hd zO|k8L5LTaQ+1VZ_WcM@G{1nu2aX7QE0w^jDGZDfRB{+^P?jCYL;p8*DhN2TqQIg7W z=J3b6Zve%z(@eF56swIdcLR#+7nulQ3eJ^t_m+RmfWkClRSktIO>x=UALk%8bnhsj zkc(xi`6;MQ+xTGk7@+8rz(fdBaPEsu?B4wiP{f;js-fsiQw(0kISWgaeu8%oR2MMS z{1nu2(e(YiyFl@J851E)!MW2q^m5fzpqQNGepz%}sL>RfzP`8?zHK(lhP7}?WvV5l zC|fyTI#86nVj_eoWbqzlcG1?~K=Ii=qK2XiO;K--H&Wam72F6YrmthF`6;L!6}In^ z8Bn-wWg>(rnjl5Pc|nbU;(Lp%8j7wo#V)sfNU>{=O9W5^wP&jNDX1OQwqdI`KvCG0 zi4dkxLJIqhExrTAh)t$fL_4Y*P0{3_D^l3l>7ez(U72cr3aZmO>>C*X6n*wH5yBKL zkwQlA+ccn9G`X^dqB~8YSGFA~u5>^16xQO*Y^GX5iu9|qCjv#B1rs4mp@I|*4llqf zR*S=(u8P*82Tk!v(*-HKjkn`@)KovFT0)A%5nFEoMgDmvLYSgGQbdik7z}F>k~*q} zLY=0Vy~iFYnhfc#4{P!26;sVmK^@&UqaEyl;&VO|Axxo)6bEGct^kToTl{J$deRjA zqZT2>FT;0sKrw$8Q_W979o=i*emf2n*$0>iVT!IuQU60uB2bh)u?-XLC=HrI`}{1V z$oshQ08l8tWU3{kDE#p70#J0!VZ21JBFpgy+gef$U!fKs7UL~bYh^(Q|q$$>P9D@|=(w@$OwRk&&sg{u9 zmEOg{KrwwD6Cq5Yg%tNQ*P$<3O_aMP+EKk}ibY%XkwP(SnFUZxn8{T0Q&7jn8}o;F zx7lJI6Cq5YgA@y5zs16F(Rb3l8j3zNg@UdYQs`g&dK)M#XED_hQY?L!unyK@oH-LA zOfeWK0_8g&0E*R%-_=m`r73Ls_d<$&o`>9l;<_DE%}+rc7jdS0V}L?+EfXP3p^p^z zR}A$7ij7yAgo}2R7EN*R1&5-;*vdknczlDYmXP8?K-40jXnU855T-CdihWZiy8uPi zN{bqbel$h$rLMRZL76elfTI5@rkbCEvfiRrJ%0nm-c3w|FvV!3c(+3lpO0$b(L6%5 z7X4|8KXDz9Vyv>~WT1%hVX7shm_8J=*Q&4wuoP+h_fx_T46Cq470V%A$MMES{iv~uOoTAS45Vm!JE9LzypUNFDLO6&(iG*+@{vw^~F-?18sK{Q1#OF5)C+biKcPy~20)%+CH zT11VX;R+PbPcRX}6lO@VH2)^vkLr9VwuWLbO)=?VeWaK?@aqMjsPJW~C8W6CBELIO z_?~4VgefeM;&OANu}7fol~Xb2Sj!k7qQ ziX}+ViehO+W8qKuCyJ|DuFcHEOE0Ch3 z-iZ#d7G8#JZisf2K20&&>I+ij*q*>&#KI>s)%+CHS{#u(bQ#uS#%v}+n8FSz_H7@D zzw6bXd%T8XC{1D1;S*A5yzCwaYaz3csphBn$=y-&jrw&2ir34T2w@5*r1*IGnjx&k z5|d6*qO}-CQ@qtJMv9ly%x!@pc|KFkPeH9k>2b>}p!m6zi4dk(gA`|S?iT}vXY9xt zis3Xx%FT~R5$jRe1}L%ge`fr>g}NKb)BeVG0+dsHf0c3Me{1_*Fx}^GcezMFCQr zijT+pQC3fxYJLi8N6l3XeF_v~UosKG6kCzPPAm8XP-r|j7%e(3M$&6xf9VZUjCz!c zPiXIW%2Z28@x}kAH&7J3WFmwqwj)J;x2vCkLLu&24aF!PiiP<|vAg$VeE*_bGE>b@ zLG7p(!G=XZF*Kct5TeG0pnxBH|w9-#6+=jK7 z{)mYXrf^4!1HI>}0Y&uHb}^#0Frq0Y*>ZN&r4cjn*Ga=@rkbCEIxb$DpV9=12XRb< zFvUKk80wN44;1Dvht*IR(-hYqzreM)dFN{r*ikd`m}-6s>JDdvCovy@V#o(3LYTq> zDQ<=7_W+92vk5g6V`&PHwXcxE`1_X^KoJ?tRP$3%zL*^sycH-~hcgkv6bF&wRNT7? zppdItd`q;W#?chni(eweNSE(hfWoZdlK)prNU=V#?M0w)Q(z*5DLj$FTjK#fXMVug zy=ENaX$qUj=SZbya)7Fq8u)lf{J zDLM`092WL-`iP)w#NGz>FvEhf%dG#u7q zTpCj?A;o3agH2&AUS%>7!W1WvqVa?MNkHMzul;S&j+#PKSQn-u#iqlbuK`8pAxt$t z1$CeHuEzVvKrvt>6Cq6DhZGq-q|tg)?k=vOm`YQenw^Fe`8s!GfFdl3sg{u9Ld%?? zK#`luLCgP%B=12q>0HXaB!iLW&KIn(l?Q7}}VL5T*!43Wb2j&tWZA{#K6@?I;tP;=5TQ zQv5Nx_yZ_({JYE+f{$O%4^19N>hxNy^j<* ztAlcZqQwcOnxBHY?sm93_5iHKy>m>2Fhv+reCXlw5!T|dYx{W7TFj*>US`B2#ffNF zboa~sOf^5nS8k_mI=gEEQ0(wwB7`Zzkz#M>avh+EGBl{6Frz89Roz93J?gFT?W*8O zOf^5ncP@p;U9||H=rfy%5T=Mkindo9xB-RV&+Ro7^Jt2=X&j1!F^%vJ=N#!B|F7n! zDCJUE=q0ZLilg#OgfPVoq^R%Ziq{u|+6=xY+EL~-#s1hkxE9WVk(~GS)R=013hKC+ zbd5JEGt-x5nJZgP*~6uM;_fqiaXJK%R@_fvz_d+jRjMOvmRIzAvJln z`)9eDbd~&=dj*hJwEXlgg%+&7rC{xiOjq~pTo@P0va zx9%b?T(_?Jf9E_Q;>u2R=onV6`{fZ_vyxeV@Jq`Y*s<$rn zl52a8O|AViny(hT!){n|_ht7{fAcQy=;kLE8O-KM^c~Gu#pt1MIGZReSPe~KOgI}U z*c2??aQXrt3|S3dRn@=Arqu0AnzI!0u;Uj^>by+UUov?<$JOdnPdtX-UwiBUT_w_# zE$45Pl)0L>n9vdCWP=r(Avh;4Q8*`Ob{Qr@$3HRLr*V39qfFuXeo}gWWdw;F?HUXq zwqx0Heh^Asi}p@x4j|go*bG4ssR{(yIB4Q}0QsF7W+4KJt3R^EmU0X+t!l1aQM~^^V@2;~!lB$auUR2ht zC{WkUJ8Ww0>atm;hUdQ@$GZoBy3!x8c@n8B)LIe(+1em)=b zDry?c$(c)RhTxpI=;53ksc$(5K+bjFIx?eLT|qpbtU040g&=QpQgZ;rpg&vA4??Ny zm~=@FfczQCW(b1Bnj*-3uf#L}@tq=nSp?$8^No=v>802V_k|3c3Lq`#vgQ0B)QW_S z+gAc0BNnn5f*?}s5X5kC;$!F*MyvZbdRDz6!D`~v6}Vd;ky>5(tPeRQ{8 zbKtEm`q2Z_^=dbpQJcEr4msn9VmHg{0b1ojHcgDW3?J;oexP!B)@o4KyW?!oKdI}= znmz@fF5k0kqOiI?sdPrfB|Wt9s!J+_&EZv-)cUaq7x#Hf1gJ|joQKgNj zO{tr@POkFvhDMnlQl>2GFP=Ofof(qx9yfT7!p8N`RdOTPa{fk1-JbHF(!LqYNpLKi zAvh;~?zp8sWeyDikf(VKc8EaSc)k)M=U;#zoe#Ra1dyg5*>Zjm>hjUzsr(86nOMeV z2!a^+B8d0$-&p{%Xwvl@5l9u!$0_t3^AIFoI@%gQx|y=&{2 zY({MkHs3Z8M|3oyr5vE?G-uPqIQU@v5*$|0hLv96;Hq|P(7!oY#@rSh?A?V;6n3z* ze-(~w?z|27<(zZ9*c@I5t1nW+h^*NfM&RJ<+H6D}9W1>ipl(n`*lDVMNO_Ve9v4CZ8FOEyDrPE?a{PA2%@wS$gt z_q0=s7u6mAInVb1ervQ=nE_2 z8RMQR0*U2$H>)R6E;H-M5+QoT6WX^1#K_0 z&-t$+kR8LtIe4k|cRczNS9J0OB)uVQkO+x#aLz|n%qXz;-vJKpH3O1sS4sH_~-4h%f>cnQz4)*$*kDGDP{TV%A#jG~4 z^@S6JWi@?DvJlQ;n z9PH-B`Accf4xc&>P2uA)Hd3%Ds8`|iNu@iMf`bR2VN>enVBK{u+d`w%jA~>2vRYkX zJg;EK%G|+Y*yY}U70^}o-)GDD8zptIqIz!r9hj3AsceSeoTM}u1HbEbdOYYXfS7KP zJ0$`+&-2c#^5;YZQTFO`5tQxgSY4*tcW`83#!K&kx(55P zIlSsBpIM2IS`>}h_5jqi^E?|-M|BM;QMLeeEeU0_Xmt%eAd8zZ)LYpbR?I7strw#% z0|P}2vU5857}WLa78@j~x~So~3^b?$bu~#~Q)^e(OurCR*Z8LkJAk^PQrJ9+)U~i{ zW8Cuw5$D@LcXNNiMhZ5Cl6E+KL48v)L0zS}Y)akK^~iRR3^dB;k&Rc1`ilq8Td$7? z-bbFJLCX(7S9x@YE$45Pl)B85T=5rK=VUfRa83-`QEs>}c+G@dP&5Tr8C8!zbRrmz`;AUVAdWT1S{NdOXY<2gkngf)yE=%w`CJglZ#5(o07(XnWbZx!EF+R5Ni7ZW3?{b+CN; z#C?F|Wyl7Jkf_T?51V?Ju{OlyB{+D|L^h)~2bX7c!bvvrA9EGZ4$oxM#5h=M^gtYz z$D{-&aPUOQ8*7v;%&Jyb8qW(!m46~}VH`|~hXKf} zjchqT2-WdzddNHi5DzytLl9)jC+ zh&np>^umN}aIoGTHj8$!?acwW8M7AL9SAFyk;vAIad5+OeGD>*?Q;|y-1P|?B&ma` z;VGAx{{{z}zF<>pckt!bS?J*K_+baZ!FzJqJc%6Kz;g}m`6-ho9)hN@|qeBWf`1cn!rEU(^o_{qH8l^?rihZK~61tLZl!HvKA&=>#;rM{$ve#@mf1{*2 z{yX0puVGGx6|otDa}sNcb0Sq)stq8Ia_h^zuI~8$_B6=a@lgo!*(v@!w8!L+Y&ky& zbuw{1?a*WZ(f`S22!ePlK#;s-^_BpVdOBjN2*k&k2B~VyIhh!%I9eG%DudZ_eh^As zYh|0Z1CWFWHbW4^av6dg|1)tLbc-1?KP8GlZmkigu8d6q*hTlNgqZ`yW*Z-dt%3YXbP&SY@}dQsItT9yLd`E1k~mJ zicP7Tx^(8w`~!`0xZU*UqWG6Dd_B3e#7%C%GVK!)BFK`m9a)Yj+-@@D@02w6dY>n^@@>TM-0BS$v$LK!6>G|zKr6T| zRP_a_f>I($L`7Xjwo-YCUbr=PXZPWMhcnfB)qFYQUqnc4GFG3&c|ch9*~cG1H?UzO zhi=vU6-yV+cUo#|(|5JC!$^7Add;QO8mv0j73o5%?qopMJkXR#`X?PbDim%6?{ril zB87eYr2IIJZSKWH`~bv3RU(L2$Zd2U;=~uM{62E%--)*z`=wa!Tu&mSjz%73pP1V3 zZ`ko|+uq4->qmsqO8)$wbE4}Xm39KwEklVNArq?oWlJ_c16LtT7(4=Q#UdH+-5oqTBqldek(=oCew|l zbymE-j7Ygtymmp8Ipj#x^S4e)1G?+v5@B9;Y$QSi=OxmOGcO$!9Dt?3c+fQwONu)W zOU$YuWGUsse6xOr)&0{$8{#Od3;UxT891+B;=MGX2aNZoU3k2#G&&sxujFj~)#H?PHA8TUQ30@m%pYa&35o@IA$!K95h-e`iJ z`@AKRB-JxDz=1)jQlRIAPeg3(de%^Gj(T1qUwj_)e5H&Clt|C9(wrNlPE&)tp+%Jc zCNc$E#2Rm$!osXPOVINrneS|isGFV#g&FOJ)~V{fs$A4=f_!*dr~U`dRa)nc>lQ$h z@$@6=C2XB0lYAS&yo?GYLIme!jt|a@nv;?futYW}GS077nx8KZ%fxAyusMB{x`7`! z-L6E`^RrOj%x_dET>&g%?THXU76V^oi80#L2v~OhRE!j{c>3|MWP9E~mhdfh?_gcR z8Z>3=`B|uY*rpfN_5;gtMIuCyMcofsTHI)z0xVBVW;HITUKcli9+p=7IC^eo<@pR) zS}!8%`B|t_z!NQfHv>y|dm==TMadsou68y*3ym;NW=k&-i~CuzdR}^VKT0Bb*5pK> z(r->AiKr+&pLV~GmXHcMVGnxV)0POSP0s^*$lNr#Ah%7h2~i_pxrGc2gC=(>lFgac(@8t_>-LCez!8sF$#H z7PT6^1m;EOG!Y^=FXaI^FY)y^T?H01)u*{4mXeD+EW_^wV{?-EK653oXlfGm60*2= zO8f;Z)`N%;L6)40$kOj_b{?=)Y&>WDrdnwQmv~qvWpUQU_}+oN!18DpQP0mp>3MW$ zaARO8JVb;DvLswWme!gl*Fy6Ml^$@ahUJP_rLDWX3ma>m@7WTdGE^jz_?5P<(;e)r z_PU)WgVL_GB0_3YT4;wxxNF)>>UkbRD9uitNctzGMQBz| z1*P55A|i#A*39w{j!hxQusd9F+7Bjzc$GGSWaO>K&mCMYe_ zmXLrae(SPB^CR>%L=Wm_VjmNw%Gfu<2eC$bt z2+m9BRh$?5FTe1Lv)_nJmBMNbBuDVD#AgNK@~r)EcM!0cjwkB*S*R-;E#En@z_Nb^ z5hBRq6M-y8S{~MgbvfuAzE;E%8^y!2y+4QL%Fzq>CsGHWA?hV$(L3{^A+U4{AwmRM zT%wRgMSYeduxPw_{7l3Wc8iB4DW7wr{?8tZ9Y7V3Ebt<`&P0*h}25hBR4 z6851M3RV#y5gK%dKoR@c+q?X z=sE8J5mK9;bK*F!vHPn$91nB}Pl-4&diGq=9O=H!JoFy)+%}6y`X@chzHs>fdJcU< zL<;Nq+t@ugwuxt7;X6;CJ`zE^dX8|qf)hVvQS%v~XZasQMjiEhJ^y$s&~w!vB8=Aa ztJ5cNSC)sI@q%^pZZwp;D|x>kZ|}*uqY@f5ZUyN1qau+csh+6;hN*>Vf}S6=B4TUT zb5x%JsOON|T}naEp&f}piS*q5Y!>$a%{i^ZphYb2L1YTHh)1!w7}nt-t3c1;eTbO4 z>A7h2uOZMn?Yfp4i`vckyF9HkZZk&%->=@?46XC07E#aNI;oqePu)G5z`XR-Cqe}0 zWy)Qgm$t6q?}4RF;IgwK7T^0kES_JE;_{q*mE;C2abZL~KMQrL5j8TsH?Szi5Fvsr zgYP4YVz-+%z%nZEs$x;K20W5@SQ@_YLzd9o1D$|nTsTqB&q7^sx`gSx1(uGti4Z{+ z)g)xuqxTbE!9Sj1xJbmZ{V@;A^YzD(gsZTea{Sy`;Gb#(my14mZUb|{HwIYr}u>TDE| zB%-49?4@=AEzx~L%b}p>{JTU*ZF=?@TZv=3a_6cl&?zMoabom5*qQV7A=mN)$)M*e z=|s{$>G|NU#3jVvNkSkD`ea6W(B*=5vZxOyH^NCfffxgaM5Ctl%8@p#bl?~g=A z9rfJlbQ`=O>{CXB(Rw~yashW`VAuKjux?v_69Hnbo(~v>V$yl5HjTGG7NRBD6h`DcB+& zaax4Q;Twsd=l5NRn7Zk?gMnN=Xux~k$Je5EQ~s2vbsDz_KqSR8t?|lc^fIEJzjac2 z-dZ^*4(8>vGZ7*NN<}Pb**q++RcCNrChq)l9z=E~nW*Pyp{}0e zVj?~POH?`$BFJ(+8(Gx*sb@eVobhK){rA;-EcZVaF?yD%Z-pv+?YJi#s3PP{*d!4Z zrRNQkPoX6&!W7*>&j!tjklOSd_WT=;Y3lq%7C?8XEfFV1&n0u@FiGKR_D#_9fUZQ+ zKk2#tE7!Sj^_Zs?c z9b2Em)w9zmB8=8^{E%z7E05?590%*xZafhnM$ePKM`2Rg(@z^f&pW3PNs{WB8lcUm z@Ef4#ytzbd?Rwt+d;;oOQQF)a^jxrr2$V?AZV7+U8chs{o+c9W3D(>n7<`e5s9Jvjq!(W!1G>iJtIrGaGw zn>fO}%-cnT2+oT~9&WAG@09R2PVZrZ?ub~T3wc=nSRF%_IWr4eLc=^_Ow>!r^1|;E z-l)Gfl?V}JSyPBCZlkm6!McPzzccVdwFW}p^RT@A%(<&Ee^K}0z>-}+)bq1YSI>5G zZ7;&QB$p5&f-IKrk;UrGhOW>&&K*`;D`L6wQLNI6&(6fgI@?P@52!}_6G{9^Qyg{} zJFAJwn2Vsafx$#bZA!Z*_X)?;&#_HEpi{X<#EDVbhZ!=Mr1Q;pFevS843YFtN?Sc~ zS~)1~dOQ&+th6C2E;zRLhi%7z(gGe4LA**+ICmN+KGx$>A}DP`29Z%mrKQi?gHH&* z$|k~SrS06oxd%D$>$$nGZW)C{fEcCuoZ;N*O1dij5R}&Y6OkmT(x?FzYs?r2N^4g} z#MZ8~1%Y!=X{Ys#=7Q1`eiMNbDa~w56I9y61>cTBiwKbE!XDd#En?wEY!NR$d(Q-= zwNNBt>ZY{qN&94=b$Z_VwO`b3{6F!u&aZ97mI4{PZa9&=GpQr>ZH|v$l6<2G(=PM73QnnYetW|E*6j%ZkhgqY>^Gj4( zFBKw0kVWe&vP^5L?++{s5AB*RVsS6yVX;}mxrutJPenAa963SM^RrOv(m-*V6s$|% z^F)Xsi%J=?7*4g11r|rE*klolO9c;0{}5kXmw~-UMFGnmTcVzyg*t5-TUOK*SbDD` zLIhc)Dv+hKymTrw!mE#te-g2rsT8Yc@1f&S5*1s!s{obZOCm``MO|@zymbIA@kR4j zHt2a(9uZQTp1Y_il4^gOGSNctx|FS_g!0(x#( zMMMhg`Of#%IJRWdZ_c3S?D{9zHRRQELD4as_-P9F^FYsUrer0e#?#)1f61 zM(cTW3g;)6k$oK;Vcn|Q5dmWKY(F^__xHQXw?jbB-fBdWqWfQ`!T+LLAc)RqbIw=ed}O6Qi`Bv7FBm9~+V#3Q7yJCzAe2Y3@fX zZi3QO*AbDzO8cJVh?M7)lZSxPZfqffc$Icl=Lk;xxca+}fzqDtA~Nczw5Mfe-9c&g z2Z%6QX}#M&!){Tgbrdd*gd1Er<<5lNCNjT)ei&$SDnv`rU? z*xHqL?co|D{A5J>Wg<`_rOhsHg;$)xulCqNix_c($P{c5ryF34*sPq|5tR1o zHW5=dr9J4FWe=^>?7_VTba%k)5{rIuuB1B>EGqMo0HQd-)g zcUG{*4}*vhL6$ffT;oXzLwW?N?Ieh*$NVhQr%VQIP26W7J6r@k?;I7^4H^%An2eU`or zSQ;x5A%ZMH^2qY-vMXMb?G2ZEC1N@2BUaCE`wv4&+%OGv2P(b0M3RV#y5ek`v6Ewo zAwDBP&qm2aNNsxVV^V-)l3%|KKN&G1jffMY=TsvpOgcHj*bnqPKbuJUCq4I{{>`f3hOz`(H_UPAvh)p^c?kx2;$ZAo&lUY@hx&!D}tVj%7~0Q>iOM(bNQg>>%WOG zTF+riIX@rePmo4E2gt~CcO~x=!j@M!KY093X?hFv?5Id2NvdaRfXDs6bOJq>v?5|_ z*K@nF^{D5sE0@iLE6zThh(L+-tP|Z1_3ZOh2H#hlt4?GJwuo*`utmuHT3ZHs4(?0D z)J@OopKd*b)+w7G_Dj@m{EqOnPMejS-vuvppEL=Y%!^{8p1*ZcN-LcUY8Z6PxS+7VWk~ zh#-rnBChe*s_vn{a_);ljEKeU6c0&y=&xvWk|ybieIYNrOKa&<#4mZ$a2K}z(81+;^ss>KMQrL5qP8TU0`|A zkq8lF+0YzW8V%IJPeyc}oiatl;uj!R&p}0;zg!;KyQ&{hO!k1l_3V7fa@EkMsBwh&4Gr01yZk@lcx zwOvG{u%55Ua-PlJGD%(*^t}535yY$K0n5FSuV2eO=RwcOM~I9%>RGnl=i{K~R;P(D zTF(yGI6o17n?0le)=fE(2oQ7i++;rI#~ZWIZ}`cGxKJWVQaw`x-0-pAO3<^;bt1NQ zJwHg@hI&q);dLAIY;cDNlt|Be_4=Zo-xWV>3oT;p10qwfMfkPA7IDSiA{X==^@ND2 zo1U}#-fsu3^HF8^E>XKFInUENZ%yJni5in&(hQnRY{Lk)p1*ZcH-tZQ8;5rl-Y60w zg7ae666fXdtVTb8C8omswTPu4h=(QW7)NP?)YO&%%RU(kww|Ab(!fNeSQS`fgXTnt zAj`s5$l_>j)eu<1^!piqtybFeP#%`Dr$a?ljo>4Bqa$yP) zBFHjF1z8eplJW0%_3ZrOl!zrcjE6-nowLU_SIn*lEXR8j_53WH^BCvPhvDk^ z_eLV5Ha%Ca<>>j4deUT|TkT53iP7^MkKd>-rK^P=py#a~MAAR$dF*;mL(sFX4-qM> z=XW=4@g!u{pc9)x&x`zsAYMI3N^$hO%tE#~=vny!kx@rIe|n&61bTiLN`%pR-XirA zZ*H$0pYk2nt@U*xK#ZQ}H>!vGd-vgZJGgq*k0p{M)iX7~S*Keq0zDs2AYyCRbMU2I zsOQWumu%2;?cEd}^Rd z3}~SLgVI*tsx=TB$;l9yFN)X+*t*ty9)vRw2wwNfr?zI4=h6acc!O z4xR!mKOVGm7O{jy^ROg*;(V@p?B*zWU>W+1sOM*)l(uKLb0)AH$|phuS#&xgOMFiJ z8(>+nS}|S3az2)arExT8T{iz5>H{pA+lYF87V1>PW`)suVDaBWgb1>zt0K$eFqKBo zJdRyV(<`msV?}qxDs4giA=p@*v~*&DYWWQ!iC<~@LwBG#G?)J94NAKfM}*X-H2vp! zIHoN&4){t>x5q@B7^V66bCjlLGuaN5mh+NG`X{AzQEQR~N>eT%B88QfkYt0Dj~fPW+u$bId?#r^<+oIx4MMrPqE?+R{Hn7_GER-8fGXbeL1n4R&R3 z*>vu%#6oyqwjN@GgXiJ-KzN<@;RN}~qYePkzDP+CSCBDQv=y*1sBO6&9Cr5-4) zeHS87BBk|=)Vt<7u5Prkvje$E3BAfhO}{FHtXH>s%9iaURUeCvPG|a9-N0;k>BHX5Io8^F?0% zA{LJX9+sbHIPaut25a^OmMQi`JwMBTe*^+A^{icQq0-9M5+Q;tN?nmfanDzLzt&b^ zxBT~N4Qx;5VaZM7{PryU!K4US;@cDT{4CU+fP>ct;P17K)rk;6mImFC#b$E|emt_? zT~jL&OZj6S7O61KsmA*)tMNvCb}~^fAxoK2iX!YW!we!skfpo_vK+YL^BfxCzDE5o zi&%1=h}E-RzAj4Q?#Fxh@Sa->B1uF=U2!fR!qM|P)B3)k=l$)7klOU@V#2xN9P+8+ z70}srA>!(y=keD~et@1Q_9l}4NzYA7IvatWrwkw>h4noA9_NPeeU;&dK+ipg5kb6q z-Z+!LL#ln+h}unYDo^Y5^x(Yo za-*eD254Z20a4H2I;kt0Grd&sPag~(PlO20OL0$}mo3M>*4#gu??!&K?t(s^g@>6lmYnUc9Y8`Cqyi1uXtGO z9~{85gq~Lm@MWZ!Xri8EKy#^*l3{2&46!YNvp^Qmu*7Xjr#H2}FPxJ?re@ zJdbe0qWlWzdCX%XNm4yi15CGAU<-Pl^^AzEUC+In9YZ}oy!JK#^nCs`5h#(Ko%@bL zJ=>k={|8z`-diG5utivD;bNF+f7tsYCaFN&zU5Fvu|65S8y<^1Rt(ZJ#pdpB0Z68e^hLp~ER=@2G*sm$RM2H}Z{{Upk3LA*;aPJAIbP=)meCA;G748X@lo7iV5J zz^mu2o1+^7RdyngB%-49+^;|9WhANek9R@OC!P`^wduLlc+UH^c4ecF16_|yB2J8+ zU0O;bo!s>9L7?aUZ;7OT((|Ng8ht>|A)kpzVLe}~0czp38a?ffDJt{bV!Lv---p zkD)~v^d~X}Tf}Qli%$M4T9<<*jUl zNiPZ}R)Esto)Ss_q_htw?zRJ^-FQhv3M;L(Bj=ARtecW>AC#7nM+EUIZFx6;ocIa4 zk=x*kbKD0aqmD}R8+%6sly>|Z5k@O*M>I!7GxTQMhIQLnK?I0V+8|xdvlYJb0oy@o z+4X+0`&&|_Q3GtbN6QV-SI?o+UIf?s4odskj0luSX*+w&N2N{f(snhp zi0*BPOu-hRI~-fY!~+}hL1}9{6ESsD+Ksd84?^qoech+MsNEF(*KVlyYonFUVe32+ zbr?TXKJpz=&)+(!mtL~E57mWv$^Alv2+qqA&b-{y*T$!6!^_qj5wT=$txvN&bLL#7 zT?w=fg@&0Sy_T)#XQ4E(b+tD>Rhz3ogb1>jjleZtHR<;$SeFKajB-URDXu&$gLZO$ zdv+LNTMR7YMiTY>EYzunRmXN|ur4wah!8=RDWj0(&%2y8z@pOm=G@9^rN!;$VJZ8} zxvO!)e0L_WX!a)R`B|td&cMu+VZd^D5D_BCGTIPXUW6=0J)cesaucycxYrkV#d-1R z92D4mwek%>)i;+&5>ZiCoNC#=Xo+QtJNtv4!`=}gwduKq&o3NPHx~<8X!9+;5^-Ym zy!>JlOmcU)uLyeXUPUDRlb#Q^nYaP;Jh?$IJN3ePz8L3Uo$- zAWppeg*o^)UwxH{j5_LBrlX=h=vku!5k~8I;R23SwM%B=jr@SFM1UAQmo?=)C$j6| z+8EIDtX@QtqDlG-rIXM)_nA~(61AJieLStR#b}PwQiuP* zn{{8ThKWSufW_f85hBQDhC5Jv`ZdZQJ)f&~@oY#EH@KSQ|DemJ#YE;E+6!KaXk?zk)Bh(EJHn8ZrU^zT7=S6lhw$51#CYu0@x53D05lhu69+t>coI3&WXB4*si^q7P zo}Yy})i{>CU<0rWpFxBOvQ$k$mcI9!Cj!gK_I>1jSFcOSSsoVIN1S!h)oFkqS2&_h z)bq1Yry4(1Z@dDQ%Q{4eAWO+KWJzvYoCHl@evz89h~>luu}VuE!1>U;p-p$ZSL5$M zB=IXP#3vFvtHXn{Nuad!bwo&QN{gH;hetricFo=aoy;~OPK?s3c5wbAdzTCKxE^ z(gt`M+kn!>hY(@3(gH4VM09n>Ui`%Wm~bLMjMDbr<~(BexXGkSP@2b0B1uxEQ3Kp# zv7;j>ZT>wXwsxf*FpWf|m0n*_0ZMavOaw}#w8S2Es5GslQx4D~?migb1=cpM`6DdFj}r!17{lpz)t-r6phHVHv*vJg!U1rTm4!@^cYU&(A`g zYTW-md?~P$+Y=#zEXi|_#p;6AeqhO-bN-}=CH5K*OS27}Qw`_d4nu$?X(3V1&qA$B zv0k@(z%t*C2oYq7HAR-Wp0#Ikq^f*SWecp^io-;J7(FlZ;XE#Aa(w(#(6jpqB1uv`Qv>XGNO>{n z`SLj;wst*#>K%i6j8zgb0*K&mQZxqMqlCI*K<7Bf^PH!4{!nhAqO>J`=y_ zR4;~zshghTVvpe!=j?at%2I*?c&UXWf@X@T> zzladQc?sjpOKj}VPN0E#I`MWQmh(4xSo9ZOLYAMQTPFic<1s`%KMVES^P79=yFmk0 z(}@s4mhTJ?0V5!;*aFGP0PML~H_<<`qOeKMQrL;au{*KCrB9 zIE)>HAdANWWRaZ{_yQW?6Sd2hB9<@r#OgWRb}vfe)T1)|RB5|DM3RV#(zDE$duWOD z&n=q6)${&=L`ZFVRtQwUnbvIM{SxS!3@75m==pcYjvPAeJAt6*@5V&ZKk2zw{^X&c z=Xz6#NMSvjt~iQg8-1p4Jm|Tb2@%Ar=lQ4OaN_%_XxW3F$IT})>Zs?U9Xl?7o|`Wv z!e~9~RO;Zalz$qC-$Q<7O9Y6~vy2_*(MRhu3#&lSDyxYkN%c$(&|{8febBSTW+JwB zJeip3K8CCOQaN=AAw;7+A^!iF$q(N(1UeGnIj*X*dxg z$g*Y;vh><<5C4Ac!gpINL@dQAJS@|Pa`e1%$y~f4{QVnI&(A`wOJCipWx(>fiU<*8 zS+N9J2LC)a4A!N){h|9JmYg&mmch+}ab5J4w|0PaQQ1h;OUUAU*sTg!2DuR-f-IKH zkVWyXeq#_=^FCuUL@Z_hvD7p|-rs8%$R9vSY_>cW093aJ6G!5+xA0y8tB<;ArZu@=O>!+IPp>WG3P+f4ObEwb=0%d67Oixvyn3qM(a7$ zZwT(nc6$3vVBOL-5CLNJY&nMW80XKQtK>k>t+o?MlIocnV8y&4JwVUb_Y$$S>)9YL z5%oN&bx|?sd7l>%D3PAuS?okTfByE_30lOh<3y%li&({J5feHzOv9Vo{zOdO^t>|i zJ6>@%J*?EXeznrlp7XR$?cto8s0O+g_!9rJQ$#&~>!f}c99_>1pF}-$kq8l-ml-Q? zlXmH}89%d-9prXY#FCK7!&0isIn@|!+13#nX6s0zo}Y!fi8^e2Z~X1q_bw44$YN}R zEFUMfo(@VgKRNW1h$T9Qhh_Xi&W{dne|Y2H6>b(t)bq1YryBWBUlhT9U2>HO5oFP~ zLzci%ihZGZd`)RQp+WT?%gGa~vpNKdyO6zp33u>dA=hQi%G~)^)>7SH#uf;fgf#;!g6L#u_ zmG(37Bxh_}1{?#W&23Bs@hYw4&_kT~98JTXptRr?L`EHzX5leP36yrM9T7$=t)Gzr z?#f%UcV)r4HSS6Th*6qF2hL-hJyzRy2BqEAAd)0i8a2S&Q~eA;X>A7*v9&9$!2dBS zZG21&zQ9vBoCuUiX~XXAMx`y>E==s} z1+6ny-)X6+-GmnKw9c?7js^;6FT=l+>SaXK^S4gw%0}BremTs`&1po4;Jj!#;JoOX zKX3<@cSh;aA{PH59u}$9oU63Cu8AR_wCmG|dVUs4&$EnwbpaN;c|?dHOIIgksc`9k z2w2+MCiZDqtpT569+rwi&P~(?hF9?qztU{KP2d%FF zOV1D8Hi=l=OL$nim0m|_^c;R1KTW7uM$}8lVtg^m;ygeIsThC!cl8B1B;&f{B0xglG=!$=m{oEKLq&7XzRc(PY zeJpKhcWCnsClhgE^lZOd4e8_$W)^{-2bvH`|D@-K+jrs@oxJ7~k-~bOcE%6K=6la& z4e0sxG9rjq&$otg^sKd64?h{P(virhqn-zk?J*tnTz@?gM(f%ABj>fUfLoc4VJ?(f#250-z_SlIocn;Q6$&xzbkwugpu7mU;*2{& z1WKf5naKU9=S@~!9zcuea+=5#Y!UgK7U7aO#0K;{>H-l{H$8WI(qsp;PKWbWe?;xZ zrIe?2%GpQY0X=y5x0}#pQo@LO{?3IcA_#~amp_x>Lp~UOzL(3Se7~vA%ZNW>yhO_(l-3Vh2|$u-x0AC zRr0W$iRIkfRvOVApLWi;NYwMQP*?y=od62HP8OE)L#`B|tvCOvHZ5n$0@LWBsiWNtIsxYhr#Y2fXcOiND@&|dbVhgiF$7Pq{{-hdJZTiLTb~qYcc2Qd403l z_~#1lmlAPe^n9~dH>BIL@zE8~^M~I=(m(0>ZTLzpxFKxUs0BOq!g_9Xi=*dz`d_j^ z&xTEjAYMIFSI=z&=3R!X=eMnij5_Lh{!ZDBpy$t>h%j2u5eGT1$lWy$*a5pTp$8El zM$da5a-QaI_%dn^==oP4B1uv`Qv+1a{W%Zx{CXe}Tf3g~++U%dSC4rx7W8a4oCuUi z&v^kyQP1(lNe`h#xEK?eLM_6jp9{8#Z&^F`fSzAXCSvNQ=cHY^7oc@+8sczB)NWF? zHK3n%w$KQ})~TKQ4xg&k9YfSh*g9?eOg6#1?4L@62+fOB5ocb8ynT2MSRUkF%@nc3 z?c`xu(Ek>)bT0jr4h_?(n5gGxp{|}M>@9T%miuKyh#*Ve7F^>7-*fPBstYGQXUbM9 zEpiVJ%MgFg73Z;_RW`tK>>^Rm&q66Jaj4%qV3`#`gb1=^ZbgP~h2(n~sLzd(n@kgK$-s!g~M8xvo zKbD$C$oqS(Y{6+1*vn;;h69z-P$Ef0Md?{@W-jVkbGpWA(6ix0^Wp3UYEN&lqhss|rZK+h4DM5M5ur)ZqTv3-5ha~bG4 zX$cX;t7pUZsW|Zk3C~MF&mZiFj5_N1&bu60&~xk>B8=9v-r%XYD}TkFI1KCd#f1nE zqvv4(oHrAa4u+ovJ^Q#3Ns{WB8sMUC+99ClIR}Z@+Vz}pHW&4r+2r|G(DSS#M4&`^ zu6OJ>>iK!6D?^|~+&@KR3bu$hoEEVpY4ZD#Nv6F zhee~}8aAhJw-Zvp(!GGFmyo42a`hWv`TCIv5oCF=16jI8tSSMP$cE#JL@aK{cv$Q+ zZX(O&!jEOZazlxz=Vzf#HT=vD`2)+VHbjUZOPnjRoZauZ8k)zuFU4KttM^#eaj{B! za_u5E)`iy_y#lJYzlbD$rD;$6fSq;ER2jS>+*0NvJDl2-CN-`N&a}(Lwst_bMv;gU zqcpF3y^!wAq~bDA+8z}m>7SHlV>kIDD6LSHh!j?uf7|mowgHYxW}q~$oZr6wDLVF`G&5Zyj810d*OVPaOvERWC*TsVc3TjE;GWOx}a180ankpf&KDf z5f;BlUOa|cI30c3wgsiYv~)qzkKHdA7v%XFgw8_oOOh{=AvpcAqXRXLkL;;HJr7yC zm4}HNR1G{?hQ%*xU58QqvS`Mo-6(#UJqbxacE4cs(u~}0yQBE!?K~tyaQbCaC#qk3 zSJbPZ#7m!bd@0y3H&$Wsi{9IPsD3%O*0}`5FY{cH^b_e9Z<$8a%l_{BBN>9zFKfC` z{lYEuz5%5qwK{jQ7wnf^t68)#OVTCHXuHgklB5@-glY91Bm;1|<1U&zxNepSqm%*n Po*)@RS{ZQLM@AU{%cG%> literal 0 HcmV?d00001 diff --git a/logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/trial_1/flags_0.json b/logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/trial_1/flags_0.json new file mode 100644 index 00000000..838d5490 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/trial_1/flags_0.json @@ -0,0 +1,62 @@ +{ + "logtostderr": false, + "alsologtostderr": false, + "log_dir": "", + "v": 0, + "verbosity": 0, + "logger_levels": {}, + "stderrthreshold": "fatal", + "showprefixforinfo": true, + "run_with_pdb": false, + "pdb_post_mortem": false, + "pdb": false, + "run_with_profiling": false, + "profile_file": null, + "use_cprofile_for_profiling": true, + "only_check_args": false, + "op_conversion_fallback_to_while_loop": true, + "delta_threshold": 0.5, + "tt_check_filter": false, + "tt_single_core_summaries": false, + "runtime_oom_exit": true, + "hbm_oom_exit": true, + "test_srcdir": "", + "test_tmpdir": "/tmp/absl_testing", + "test_random_seed": 301, + "test_randomize_ordering_seed": "", + "xml_output_file": "", + "experimental_orbax_use_distributed_process_id": false, + "experimental_orbax_use_distributed_barrier": false, + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "workload": "librispeech_deepspeech", + "tuning_ruleset": "self", + "tuning_search_space": null, + "num_tuning_trials": 1, + "data_dir": "/data/librispeech", + "imagenet_v2_data_dir": null, + "librispeech_tokenizer_vocab_path": "/data/librispeech/spm_model.vocab", + "framework": "pytorch", + "torch_compile": true, + "experiment_dir": "/experiment_runs", + "experiment_name": "submissions_a100/ademamix_golden/study_1", + "save_checkpoints": false, + "save_intermediate_checkpoints": true, + "resume_last_run": null, + "append_timestamp": false, + "use_wandb": false, + "profile": false, + "max_global_steps": null, + "overwrite": true, + "hparam_start_index": null, + "hparam_end_index": null, + "rng_seed": 611108887, + "set_pytorch_max_split_size": false, + "pytorch_eval_num_workers": 0, + "capture_jax_trace": false, + "skip_evals": false, + "?": false, + "help": false, + "helpshort": false, + "helpfull": false, + "helpxml": false +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/trial_1/measurements.csv b/logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/trial_1/measurements.csv new file mode 100644 index 00000000..5b386bc2 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/trial_1/measurements.csv @@ -0,0 +1,284 @@ +global_step,grad_norm,loss,train/ctc_loss,train/wer,validation/ctc_loss,validation/wer,validation/num_examples,test/ctc_loss,test/wer,test/num_examples,score,total_duration,accumulated_submission_time,accumulated_eval_time,accumulated_logging_time,preemption_count +0,0.5,33.56396484375,,,,,,,,,,,,,, +1,,,30.4241913576294,1.997900826178694,29.20389416315049,1.817293487181963,5348.0,29.431779469643203,2.069973391830682,2472.0,64.17994022369385,137.72752594947815,64.17994022369385,72.91593790054321,0.0,0.0 +1,0.5,32.949180603027344,,,,,,,,,,,,,, +2,0.4999999701976776,33.51580810546875,,,,,,,,,,,,,, +3,0.5,33.54902267456055,,,,,,,,,,,,,, +4,0.5,33.0034294128418,,,,,,,,,,,,,, +5,0.5,33.11461639404297,,,,,,,,,,,,,, +6,0.4999999701976776,33.20351028442383,,,,,,,,,,,,,, +7,0.5,31.9262924194336,,,,,,,,,,,,,, +8,0.5,32.19672775268555,,,,,,,,,,,,,, +9,0.5,31.687131881713867,,,,,,,,,,,,,, +10,0.5,31.476970672607425,,,,,,,,,,,,,, +11,0.5,31.284875869750977,,,,,,,,,,,,,, +12,0.5,31.0214900970459,,,,,,,,,,,,,, +13,0.5,30.48224258422852,,,,,,,,,,,,,, +14,0.5,29.93385505676269,,,,,,,,,,,,,, +15,0.5,28.803438186645508,,,,,,,,,,,,,, +16,0.4999999701976776,28.9680233001709,,,,,,,,,,,,,, +17,0.4999999701976776,28.64168739318848,,,,,,,,,,,,,, +18,0.4999999403953552,27.944339752197266,,,,,,,,,,,,,, +19,0.4999999701976776,27.477293014526367,,,,,,,,,,,,,, +20,0.4999999403953552,26.773101806640625,,,,,,,,,,,,,, +21,0.4999999105930328,26.78524398803711,,,,,,,,,,,,,, +22,0.4999999701976776,26.66408920288086,,,,,,,,,,,,,, +23,0.4999999403953552,25.773420333862305,,,,,,,,,,,,,, +24,0.4999999701976776,25.3918399810791,,,,,,,,,,,,,, +25,0.4999999105930328,25.06345558166504,,,,,,,,,,,,,, +26,0.4999999403953552,24.35438346862793,,,,,,,,,,,,,, +27,0.4999999403953552,24.15074157714844,,,,,,,,,,,,,, +28,0.5,23.21141815185547,,,,,,,,,,,,,, +29,0.4999999701976776,22.483980178833008,,,,,,,,,,,,,, +30,0.4999999403953552,21.6145076751709,,,,,,,,,,,,,, +31,0.4999999403953552,21.103485107421875,,,,,,,,,,,,,, +32,0.4999999105930328,20.549654006958008,,,,,,,,,,,,,, +33,0.4999999701976776,20.05772399902344,,,,,,,,,,,,,, +34,0.4999999701976776,19.576255798339844,,,,,,,,,,,,,, +35,0.4999999105930328,18.79297065734864,,,,,,,,,,,,,, +36,0.4999999701976776,17.816667556762695,,,,,,,,,,,,,, +37,0.4999999403953552,16.821311950683594,,,,,,,,,,,,,, +38,0.4999999403953552,16.620229721069336,,,,,,,,,,,,,, +39,0.4999999403953552,16.02187728881836,,,,,,,,,,,,,, +40,0.4999999105930328,15.096518516540527,,,,,,,,,,,,,, +41,0.4999999701976776,14.42070198059082,,,,,,,,,,,,,, +42,0.4999999701976776,13.416064262390137,,,,,,,,,,,,,, +43,0.4999999403953552,13.23371124267578,,,,,,,,,,,,,, +44,0.4999999403953552,12.382438659667969,,,,,,,,,,,,,, +45,0.4999999701976776,11.781835556030272,,,,,,,,,,,,,, +46,0.4999999701976776,11.013928413391112,,,,,,,,,,,,,, +47,0.4999999403953552,10.699811935424805,,,,,,,,,,,,,, +48,0.4999999403953552,10.740617752075195,,,,,,,,,,,,,, +49,0.4999999105930328,10.292919158935549,,,,,,,,,,,,,, +50,0.4999999105930328,9.860594749450684,,,,,,,,,,,,,, +51,0.5,9.26182746887207,,,,,,,,,,,,,, +52,0.4999999403953552,9.019164085388184,,,,,,,,,,,,,, +53,0.4999999105930328,8.879788398742676,,,,,,,,,,,,,, +54,0.4999998807907104,8.653800964355469,,,,,,,,,,,,,, +55,0.4999999105930328,8.585978507995605,,,,,,,,,,,,,, +56,0.4999999105930328,8.23762321472168,,,,,,,,,,,,,, +57,0.4999998807907104,8.136585235595703,,,,,,,,,,,,,, +58,0.499999850988388,7.822683811187744,,,,,,,,,,,,,, +59,0.4999998807907104,7.802002906799316,,,,,,,,,,,,,, +60,0.499999850988388,7.543367385864258,,,,,,,,,,,,,, +61,0.4999997913837433,7.441073894500732,,,,,,,,,,,,,, +62,0.499999850988388,7.382068157196045,,,,,,,,,,,,,, +63,0.4999997615814209,7.429420471191406,,,,,,,,,,,,,, +64,0.4999998807907104,7.253262519836426,,,,,,,,,,,,,, +65,0.499999850988388,7.136641502380371,,,,,,,,,,,,,, +66,0.499999850988388,7.177666187286377,,,,,,,,,,,,,, +67,0.4999998807907104,7.106010913848877,,,,,,,,,,,,,, +68,0.4999998211860657,7.033105373382568,,,,,,,,,,,,,, +69,0.4999999105930328,7.005535125732422,,,,,,,,,,,,,, +70,0.4999997615814209,6.965167045593262,,,,,,,,,,,,,, +71,0.4999998807907104,7.04483699798584,,,,,,,,,,,,,, +72,0.4999999701976776,6.806606769561768,,,,,,,,,,,,,, +73,0.499999850988388,6.858589172363281,,,,,,,,,,,,,, +74,0.499999850988388,6.825179100036621,,,,,,,,,,,,,, +75,0.4999998211860657,6.772513389587402,,,,,,,,,,,,,, +76,0.4999999105930328,6.787856578826904,,,,,,,,,,,,,, +77,0.499999850988388,6.7225728034973145,,,,,,,,,,,,,, +78,0.4999998211860657,6.724231719970703,,,,,,,,,,,,,, +79,0.4999998211860657,6.6959428787231445,,,,,,,,,,,,,, +80,0.4999999403953552,6.6218743324279785,,,,,,,,,,,,,, +81,0.4999999403953552,6.635696887969971,,,,,,,,,,,,,, +82,0.4999998211860657,6.619246006011963,,,,,,,,,,,,,, +83,0.4999997317790985,6.550009250640869,,,,,,,,,,,,,, +84,0.4999997019767761,6.496662616729736,,,,,,,,,,,,,, +85,0.4999997913837433,6.437870025634766,,,,,,,,,,,,,, +86,0.4999997913837433,6.407183647155762,,,,,,,,,,,,,, +87,0.4999997913837433,6.345399379730225,,,,,,,,,,,,,, +88,0.4999997913837433,6.315845489501953,,,,,,,,,,,,,, +89,0.4999997019767761,6.29259729385376,,,,,,,,,,,,,, +90,0.4999997615814209,6.219762325286865,,,,,,,,,,,,,, +91,0.4999995827674866,6.182476997375488,,,,,,,,,,,,,, +92,0.4999996721744537,6.124574184417725,,,,,,,,,,,,,, +93,0.4999996423721313,6.103853225708008,,,,,,,,,,,,,, +94,0.4999997615814209,6.086005687713623,,,,,,,,,,,,,, +95,0.4999997317790985,6.056469440460205,,,,,,,,,,,,,, +96,0.4999994933605194,6.050701141357422,,,,,,,,,,,,,, +97,0.4999994039535522,6.003434181213379,,,,,,,,,,,,,, +98,0.4999999105930328,6.024650096893311,,,,,,,,,,,,,, +99,0.4999996721744537,5.987350463867188,,,,,,,,,,,,,, +100,0.4999998211860657,5.999506950378418,,,,,,,,,,,,,, +500,0.4999997615814209,4.668478488922119,,,,,,,,,,,,,, +1000,0.4999997913837433,3.099266767501831,,,,,,,,,,,,,, +1091,,,6.629959117090944,0.9415415166935583,6.5073090120805706,0.896519094288611,5348.0,6.470239208981225,0.8993154997664168,2472.0,1510.1516211032867,1629.4568991661072,1510.1516211032867,116.69307732582092,0.0918250083923339,0.0 +1500,0.4999998211860657,2.65515661239624,,,,,,,,,,,,,, +2000,0.4999997615814209,2.3312275409698486,,,,,,,,,,,,,, +2500,0.4999997615814209,2.2783656120300293,,,,,,,,,,,,,, +2694,,,5.42037396233919,0.9354544522451448,5.29343221004119,0.8926085067349008,5348.0,5.134036066843022,0.893242337456584,2472.0,2956.759863376617,3124.094222545624,2956.759863376617,162.41574835777283,0.2287623882293701,0.0 +3000,0.499999850988388,2.140624046325684,,,,,,,,,,,,,, +3500,0.4999997615814209,2.0026023387908936,,,,,,,,,,,,,, +4000,0.499999850988388,1.9127516746521,,,,,,,,,,,,,, +4397,,,0.7364908717024884,0.2286318675092142,0.9592859575924252,0.265215082315454,5348.0,0.6292866172817602,0.1967582718908049,2472.0,4402.464949607849,4618.085517883301,4402.464949607849,208.2484560012817,0.4177141189575195,0.0 +4500,0.4999997913837433,1.834662675857544,,,,,,,,,,,,,, +5000,0.4999998211860657,1.786255955696106,,,,,,,,,,,,,, +5500,0.4999997913837433,1.769141674041748,,,,,,,,,,,,,, +6000,0.499999850988388,1.6621299982070925,,,,,,,,,,,,,, +6152,,,0.564567527578269,0.1829357134762642,0.8027857833534258,0.2273161782455462,5348.0,0.4934387904058326,0.1585928137631263,2472.0,5848.90784907341,6113.109282255173,5848.90784907341,254.4990930557251,0.4602918624877929,0.0 +6500,0.499999850988388,1.664108157157898,,,,,,,,,,,,,, +7000,0.4999997913837433,1.6913143396377563,,,,,,,,,,,,,, +7500,0.4999999105930328,1.6519166231155396,,,,,,,,,,,,,, +8000,0.4999998807907104,1.545139193534851,,,,,,,,,,,,,, +8069,,,0.4789078483051961,0.1571574272439304,0.727484416597599,0.2075218461835562,5348.0,0.4325849651590425,0.1398046026039445,2472.0,7294.322849035263,7607.926026582718,7294.322849035263,301.5002896785736,0.502281665802002,0.0 +8500,0.499999850988388,1.5158309936523438,,,,,,,,,,,,,, +9000,0.4999999105930328,1.5776087045669556,,,,,,,,,,,,,, +9500,0.4999997615814209,1.5517101287841797,,,,,,,,,,,,,, +9980,,,0.4124300958709272,0.1375417541322425,0.659895283931083,0.1892048471974122,5348.0,0.3792980343489902,0.1227225641338126,2472.0,8740.309307813644,9104.06256055832,8740.309307813644,349.25259923934937,0.5435240268707275,0.0 +10000,0.499999850988388,1.556360125541687,,,,,,,,,,,,,, +10500,0.499999850988388,1.552520751953125,,,,,,,,,,,,,, +11000,0.499999850988388,1.5084468126296997,,,,,,,,,,,,,, +11500,0.4999998807907104,1.633113145828247,,,,,,,,,,,,,, +11859,,,0.3743213390753654,0.1263983249780099,0.624480716765873,0.1779944962101096,5348.0,0.3478281360894251,0.1124855279995125,2472.0,10185.611597776411,10600.63650560379,10185.611597776411,397.0793631076813,0.7489340305328369,0.0 +12000,0.499999850988388,1.5475952625274658,,,,,,,,,,,,,, +12500,0.4999998807907104,1.4453119039535522,,,,,,,,,,,,,, +13000,0.4999999105930328,1.4707317352294922,,,,,,,,,,,,,, +13500,0.4999998807907104,1.3769662380218506,,,,,,,,,,,,,, +13641,,,0.3424478947239814,0.1163665405723351,0.5873597650755977,0.1672959011248974,5348.0,0.3311809431253629,0.1079560457416773,2472.0,11631.358527183533,12094.00501012802,11631.358527183533,442.1850917339325,0.9710085391998292,0.0 +14000,0.4999999105930328,1.4168156385421753,,,,,,,,,,,,,, +14500,0.499999850988388,1.4632407426834106,,,,,,,,,,,,,, +15000,0.4999998807907104,1.4378657341003418,,,,,,,,,,,,,, +15423,,,0.3271395540038624,0.1115853888867903,0.5830462216037523,0.1638970694732776,5348.0,0.3212276296051358,0.1041577803505778,2472.0,13077.367248296738,13588.992755651474,13077.367248296738,488.7829692363739,1.0474531650543213,0.0 +15500,0.4999999701976776,1.377771019935608,,,,,,,,,,,,,, +16000,0.4999997913837433,1.4258779287338257,,,,,,,,,,,,,, +16500,0.4999999105930328,1.3671022653579712,,,,,,,,,,,,,, +17000,0.4999998211860657,1.338417887687683,,,,,,,,,,,,,, +17342,,,0.3086408433911581,0.1064912498448554,0.5636925462596695,0.1607492878868343,5348.0,0.3085389581585909,0.100197022322426,2472.0,14522.733395814896,15083.918245077131,14522.733395814896,535.8872499465942,1.1191730499267578,0.0 +17500,0.4999999105930328,1.3896477222442627,,,,,,,,,,,,,, +18000,0.4999999105930328,1.3473737239837646,,,,,,,,,,,,,, +18500,0.4999998807907104,1.5041834115982056,,,,,,,,,,,,,, +19000,0.4999998807907104,1.4048025608062744,,,,,,,,,,,,,, +19260,,,0.2974399627065632,0.103113144929335,0.5551025930216495,0.156558682952735,5348.0,0.3041775124604813,0.0986330306907968,2472.0,15967.952255010605,16581.015773534775,15967.952255010605,584.3306908607483,1.4141464233398438,0.0 +19500,0.4999998807907104,1.366553783416748,,,,,,,,,,,,,, +20000,0.4999999403953552,1.3696991205215454,,,,,,,,,,,,,, +20500,0.499999850988388,1.4576680660247805,,,,,,,,,,,,,, +21000,0.4999999105930328,1.40277361869812,,,,,,,,,,,,,, +21132,,,0.2805588580759754,0.0974362018444668,0.5357545098076151,0.1539226572683821,5348.0,0.2917021864313827,0.0946519610830134,2472.0,17413.341284513474,18075.28558087349,17413.341284513474,630.6906733512878,1.559232234954834,0.0 +21500,0.4999999105930328,1.342638611793518,,,,,,,,,,,,,, +22000,0.4999999105930328,1.3576124906539917,,,,,,,,,,,,,, +22500,0.499999850988388,1.34269118309021,,,,,,,,,,,,,, +22884,,,0.2689321488636708,0.094786601982613,0.5277967615876783,0.1499927581711968,5348.0,0.2878423103909929,0.0917880283549651,2472.0,18859.01353430748,19570.06518220901,18859.01353430748,677.4123427867889,1.6290030479431152,0.0 +23000,0.4999998807907104,1.3735065460205078,,,,,,,,,,,,,, +23500,0.4999999105930328,1.3348928689956665,,,,,,,,,,,,,, +24000,0.4999999105930328,1.3296666145324707,,,,,,,,,,,,,, +24500,0.4999999701976776,1.3507016897201538,,,,,,,,,,,,,, +24679,,,0.2497926942865039,0.08689176573436,0.5030040530188868,0.1429923236614686,5348.0,0.2720485495031937,0.0871366766193406,2472.0,20305.428166627884,21066.22006201744,20305.428166627884,724.7301826477051,1.7316012382507324,0.0 +25000,0.4999999403953552,1.332024097442627,,,,,,,,,,,,,, +25500,0.4999998807907104,1.2104973793029783,,,,,,,,,,,,,, +26000,0.499999850988388,1.2566229104995728,,,,,,,,,,,,,, +26500,0.4999998807907104,1.230341911315918,,,,,,,,,,,,,, +26599,,,0.2343841106029302,0.0825854914171312,0.4907958444783504,0.1400955921402018,5348.0,0.2648428930898767,0.0853898807710275,2472.0,21752.29531764984,22563.78405785561,21752.29531764984,772.3168234825134,1.7998158931732178,0.0 +27000,0.4999999403953552,1.3052759170532229,,,,,,,,,,,,,, +27500,0.4999999403953552,1.2365269660949707,,,,,,,,,,,,,, +28000,0.4999999105930328,1.2105917930603027,,,,,,,,,,,,,, +28500,0.4999999701976776,1.2811439037322998,,,,,,,,,,,,,, +28513,,,0.2216912238667884,0.07876488713568,0.4762626833433795,0.1341283252063921,5348.0,0.2557556132653719,0.0818759774947697,2472.0,23197.384291172028,24058.654323339462,23197.384291172028,819.3830456733704,2.1159608364105225,0.0 +29000,0.4999999105930328,1.189541220664978,,,,,,,,,,,,,, +29500,0.4999999105930328,1.2139891386032104,,,,,,,,,,,,,, +30000,0.499999850988388,1.3354462385177612,,,,,,,,,,,,,, +30386,,,0.2086407591312466,0.0738488271068636,0.4649183117339763,0.1311833148264375,5348.0,0.2501848405542293,0.0794182763593524,2472.0,24642.76304149628,25553.797189712524,24642.76304149628,866.6622533798218,2.1917943954467773,0.0 +30500,0.4999998807907104,1.1448042392730713,,,,,,,,,,,,,, +31000,0.4999999403953552,1.1708393096923828,,,,,,,,,,,,,, +31500,0.4999998807907104,1.2302377223968506,,,,,,,,,,,,,, +32000,0.4999999105930328,1.2240660190582275,,,,,,,,,,,,,, +32163,,,0.1984771179571353,0.0711668492426245,0.4568019236142505,0.1287500603485733,5348.0,0.2421484349796761,0.0761278004590417,2472.0,26088.13188743592,27047.72083067894,26088.13188743592,912.814882993698,2.274639129638672,0.0 +32500,0.4999999403953552,1.1879674196243286,,,,,,,,,,,,,, +33000,0.4999999701976776,1.242877721786499,,,,,,,,,,,,,, +33500,0.4999998211860657,1.14580237865448,,,,,,,,,,,,,, +33954,,,0.1903303430895076,0.0679398416715683,0.4504454867389994,0.126712692511949,5348.0,0.2380827240870378,0.074807547782991,2472.0,27534.087806224823,28542.67593407631,27534.087806224823,959.3843200206755,2.358907461166382,0.0 +34000,0.4999999105930328,1.2243614196777344,,,,,,,,,,,,,, +34500,0.4999999701976776,1.1317158937454224,,,,,,,,,,,,,, +35000,0.4999999105930328,1.1435441970825195,,,,,,,,,,,,,, +35500,0.4999999105930328,1.159578800201416,,,,,,,,,,,,,, +35877,,,0.1857410480163531,0.0663263378860402,0.4459928758225336,0.1252739825230531,5348.0,0.2357935390992967,0.0738325919606767,2472.0,28980.851875543594,30041.186826705933,28980.851875543594,1007.8453965187072,2.4333620071411133,0.0 +36000,0.4999999701976776,1.2142770290374756,,,,,,,,,,,,,, +36500,0.499999850988388,1.1546616554260254,,,,,,,,,,,,,, +37000,0.4999998211860657,1.1765782833099363,,,,,,,,,,,,,, +37500,0.4999999105930328,1.1802242994308472,,,,,,,,,,,,,, +37792,,,0.1835047636340963,0.065743533843107,0.4444097143736186,0.1245787669579491,5348.0,0.2344082178688947,0.0734466719476773,2472.0,30426.722893953323,31536.52254796028,30426.722893953323,1054.6184527873993,2.738009691238404,0.0 +38000,0.4999999403953552,1.2055281400680542,,,,,,,,,,,,,, +38500,0.4999999105930328,1.176627516746521,,,,,,,,,,,,,, +39000,0.4999999701976776,1.2017505168914795,,,,,,,,,,,,,, +39500,0.4999998807907104,1.2096292972564695,,,,,,,,,,,,,, +39667,,,0.1831902371832248,0.0655492658287959,0.4442900231690777,0.1242987495775599,5348.0,0.2341228920010968,0.0732435561513619,2472.0,31872.2521109581,33031.37944364548,31872.2521109581,1101.4898154735563,2.810614109039306,0.0 +40000,0.4999998807907104,1.1945302486419678,,,,,,,,,,,,,, +40500,0.4999999701976776,1.1244745254516602,,,,,,,,,,,,,, +41000,0.4999999105930328,1.193238615989685,,,,,,,,,,,,,, +41448,,,0.181653533881753,0.0650581994592873,0.4430559774179978,0.1243373726645102,5348.0,0.2331992437012065,0.0725732740235208,2472.0,33317.68475842476,34524.58527302742,33317.68475842476,1146.8406782150269,2.902109384536743,0.0 +41500,0.4999999105930328,1.1433868408203125,,,,,,,,,,,,,, +42000,0.5,1.2393790483474731,,,,,,,,,,,,,, +42500,0.4999999403953552,1.131496548652649,,,,,,,,,,,,,, +43000,0.4999999403953552,1.1226806640625,,,,,,,,,,,,,, +43232,,,0.1815308830480854,0.0650528031255565,0.4440422035111513,0.1243373726645102,5348.0,0.2351312483869927,0.0738935266995714,2472.0,34763.08761382103,36018.051757097244,34763.08761382103,1192.4994394779203,2.9804887771606445,0.0 +43500,0.4999999105930328,1.1684746742248535,,,,,,,,,,,,,, +44000,0.4999999701976776,1.142064094543457,,,,,,,,,,,,,, +44500,0.4999999105930328,1.1389905214309692,,,,,,,,,,,,,, +45000,0.4999999701976776,1.097764015197754,,,,,,,,,,,,,, +45155,,,0.1830968008407454,0.0654899061577564,0.4461929171061884,0.125515376816492,5348.0,0.236287220143235,0.0737107224828875,2472.0,36208.82337117195,37515.20438790321,36208.82337117195,1240.5367317199707,3.049771547317505,0.0 +45500,0.4999999403953552,1.1659096479415894,,,,,,,,,,,,,, +46000,0.4999999403953552,1.1731445789337158,,,,,,,,,,,,,, +46500,0.4999999403953552,1.2638088464736938,,,,,,,,,,,,,, +47000,0.4999998807907104,1.204919934272766,,,,,,,,,,,,,, +47071,,,0.188176462267569,0.0679668233402226,0.4556509749535363,0.1270313329792883,5348.0,0.2421711178947029,0.076066865720147,2472.0,37654.81112194061,39010.60394477844,37654.81112194061,1287.16175699234,3.442328453063965,0.0 +47500,0.4999999105930328,1.1755801439285278,,,,,,,,,,,,,, +48000,0.4999999105930328,1.195659637451172,,,,,,,,,,,,,, +48500,0.4999999701976776,1.2699499130249023,,,,,,,,,,,,,, +48953,,,0.1970463188319217,0.0704653258576123,0.4728940136345941,0.1322551054893062,5348.0,0.2494624653203432,0.0801494932260881,2472.0,39100.52050042152,40505.21326804161,39100.52050042152,1333.608855485916,3.5180375576019287,0.0 +49000,0.499999850988388,1.2973856925964355,,,,,,,,,,,,,, +49500,0.4999999701976776,1.2614086866378784,,,,,,,,,,,,,, +50000,0.4999999105930328,1.2385014295578003,,,,,,,,,,,,,, +50500,0.4999998807907104,1.1713855266571045,,,,,,,,,,,,,, +50743,,,0.2117411289058682,0.0765955609758729,0.4953008448236889,0.137372664510211,5348.0,0.2632191996257823,0.0830540491133995,2472.0,40546.38946771622,41999.961280584335,40546.38946771622,1380.0854613780975,3.592219591140747,0.0 +51000,0.4999999105930328,1.2598899602890017,,,,,,,,,,,,,, +51500,0.4999999701976776,1.330693244934082,,,,,,,,,,,,,, +52000,0.4999999701976776,1.1846530437469482,,,,,,,,,,,,,, +52500,0.4999999701976776,1.2395654916763306,,,,,,,,,,,,,, +52512,,,0.2245042673432175,0.0812148226494919,0.5048763864558218,0.140742528846618,5348.0,0.2709780923366023,0.0857554892043954,2472.0,41994.09754395485,43496.8531627655,41994.09754395485,1426.869449853897,3.67100977897644,0.0 +53000,0.4999999403953552,1.2876075506210327,,,,,,,,,,,,,, +53500,0.4999999105930328,1.3018102645874023,,,,,,,,,,,,,, +54000,0.4999999701976776,1.2875375747680664,,,,,,,,,,,,,, +54430,,,0.2504103457689728,0.0901349623066088,0.536555754125226,0.1499251677690339,5348.0,0.2861785689802568,0.0912396157049133,2472.0,43439.65407204628,44993.551918029785,43439.65407204628,1474.8302474021912,3.752428293228149,0.0 +54500,0.4999998807907104,1.3208173513412476,,,,,,,,,,,,,, +55000,0.4999999105930328,1.332390308380127,,,,,,,,,,,,,, +55500,0.4999999403953552,1.3168715238571167,,,,,,,,,,,,,, +56000,0.4999999105930328,1.3006612062454224,,,,,,,,,,,,,, +56344,,,0.2633641485872983,0.0943710842853365,0.5419323418977295,0.1515859605078935,5348.0,0.2942602647751468,0.0928442304958056,2472.0,44885.20058083534,46488.41561603546,44885.20058083534,1521.4352769851685,4.077461004257202,0.0 +56500,0.4999999105930328,1.367113709449768,,,,,,,,,,,,,, +57000,0.499999850988388,1.370246410369873,,,,,,,,,,,,,, +57500,0.4999998807907104,1.3996403217315674,,,,,,,,,,,,,, +58000,0.4999999403953552,1.3878111839294434,,,,,,,,,,,,,, +58225,,,0.27624290742195,0.0971394034892693,0.5533760473961473,0.1544151016269975,5348.0,0.3016079162365314,0.0960128369183271,2472.0,46331.20462226868,47984.02299404144,46331.20462226868,1568.6045529842377,4.146382808685303,0.0 +58500,0.4999998807907104,1.398287653923035,,,,,,,,,,,,,, +59000,0.4999999403953552,1.3494956493377686,,,,,,,,,,,,,, +59500,0.4999998807907104,1.4003089666366575,,,,,,,,,,,,,, +60000,0.4999998807907104,1.3520879745483398,,,,,,,,,,,,,, +60022,,,0.2832823490652205,0.0994382416586171,0.5648793532279737,0.155448269202916,5348.0,0.2993794710545842,0.0962768874535372,2472.0,47776.68691539765,49478.376770973206,47776.68691539765,1615.0702879428864,4.230457782745361,0.0 +60500,0.4999999701976776,1.314428687095642,,,,,,,,,,,,,, +61000,0.4999999105930328,1.3916306495666504,,,,,,,,,,,,,, +61500,0.4999999403953552,1.3233731985092163,,,,,,,,,,,,,, +61792,,,0.2992842805130417,0.1058275007959592,0.5677088238710568,0.1598706126587167,5348.0,0.3120878010274856,0.1003798265391099,2472.0,49223.1929461956,50973.3702442646,49223.1929461956,1661.152830839157,4.336699724197388,0.0 +62000,0.4999998807907104,1.396353244781494,,,,,,,,,,,,,, +62500,0.4999999701976776,1.402069091796875,,,,,,,,,,,,,, +63000,0.4999999701976776,1.3898613452911377,,,,,,,,,,,,,, +63500,0.4999999701976776,1.4527631998062134,,,,,,,,,,,,,, +63715,,,0.3056744732070333,0.1060055798090777,0.5747124271649589,0.1587312315936851,5348.0,0.3137460481321376,0.1002173339020575,2472.0,50669.54753446579,52469.82901930809,50669.54753446579,1708.5964970588684,4.49121356010437,0.0 +64000,0.4999999105930328,1.433137059211731,,,,,,,,,,,,,, +64500,0.4999999105930328,1.3850047588348389,,,,,,,,,,,,,, +65000,0.4999999403953552,1.3496288061141968,,,,,,,,,,,,,, +65500,0.4999999105930328,1.3781969547271729,,,,,,,,,,,,,, +65635,,,0.3088394334700591,0.1074302119140256,0.5700153322470866,0.161299666875875,5348.0,0.3139578561520098,0.1003798265391099,2472.0,52114.71427941322,53965.541815280914,52114.71427941322,1756.4026470184326,4.832993507385254,0.0 +66000,0.4999999105930328,1.407543063163757,,,,,,,,,,,,,, +66500,0.4999999701976776,1.493725061416626,,,,,,,,,,,,,, +67000,0.4999999105930328,1.450324296951294,,,,,,,,,,,,,, +67500,0.4999999403953552,1.4229999780654907,,,,,,,,,,,,,, +67530,,,0.3181459933569486,0.1100258484385708,0.5868998369060177,0.1633949693429247,5348.0,0.3225884532873088,0.1030812666301058,2472.0,53560.22973561287,55463.241881370544,53560.22973561287,1805.1806757450104,4.925171375274658,0.0 +68000,0.4999999403953552,1.4023553133010864,,,,,,,,,,,,,, +68500,0.4999999403953552,1.4216969013214111,,,,,,,,,,,,,, +69000,0.4999999701976776,1.4831067323684692,,,,,,,,,,,,,, +69341,,,0.3353715103757655,0.1148933414638094,0.5987854187857896,0.1677497223965625,5348.0,0.3283282389186399,0.1059045761988909,2472.0,55005.79437327385,56956.89923882485,55005.79437327385,1850.863557100296,4.994636297225952,0.0 +69500,0.4999998807907104,1.3718098402023315,,,,,,,,,,,,,, +70000,0.5,1.5532561540603638,,,,,,,,,,,,,, +70500,0.4999999403953552,1.527653455734253,,,,,,,,,,,,,, +71000,0.4999999403953552,1.4916131496429443,,,,,,,,,,,,,, +71098,,,,,,,,,,,56451.64101982117,,,,,0.0 diff --git a/logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/trial_1/meta_data_0.json b/logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/trial_1/meta_data_0.json new file mode 100644 index 00000000..87c57362 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/librispeech_deepspeech_pytorch/trial_1/meta_data_0.json @@ -0,0 +1,77 @@ +{ + "workload.attention_temperature": 1.0, + "workload.enable_decoder_layer_norm": true, + "workload.enable_residual_connections": true, + "workload.eval_batch_size": 256, + "workload.eval_num_workers": 0, + "workload.eval_period_time_sec": 1447, + "workload.freq_mask_count": 2, + "workload.layernorm_everywhere": false, + "workload.max_allowed_runtime_sec": 36949, + "workload.num_eval_train_examples": 5376, + "workload.num_test_examples": 2472, + "workload.num_train_examples": 263840, + "workload.num_validation_examples": 5348, + "workload.requires_sync_before_eval": false, + "workload.step_hint": 38400, + "workload.target_metric_name": "wer", + "workload.test_target_value": 0.074143, + "workload.time_mask_count": 10, + "workload.use_gelu": false, + "workload.use_post_layer_norm": true, + "workload.use_specaug": true, + "workload.use_tanh": false, + "workload.validation_target_value": 0.119936, + "cpu.util.avg_percent_since_last": 5.3, + "cpu.freq.current": 2200.1639999999993, + "mem.total": 359053524992, + "mem.available": 348292661248, + "mem.used": 7600750592, + "mem.percent_used": 3.0, + "mem.read_bytes_since_boot": 15663725724672, + "mem.write_bytes_since_boot": 91242353664, + "net.bytes_sent_since_boot": 31349015, + "net.bytes_recv_since_boot": 31352117, + "gpu.count": 4, + "gpu.0.compute.util": 0.0, + "gpu.0.mem.util": 0.0309326171875, + "gpu.0.mem.total": 40960.0, + "gpu.0.mem.used": 1267.0, + "gpu.0.mem.free": 39060.0, + "gpu.0.temp.current": 34.0, + "gpu.1.compute.util": 0.0, + "gpu.1.mem.util": 0.0309326171875, + "gpu.1.mem.total": 40960.0, + "gpu.1.mem.used": 1267.0, + "gpu.1.mem.free": 39060.0, + "gpu.1.temp.current": 36.0, + "gpu.2.compute.util": 0.0, + "gpu.2.mem.util": 0.0309326171875, + "gpu.2.mem.total": 40960.0, + "gpu.2.mem.used": 1267.0, + "gpu.2.mem.free": 39060.0, + "gpu.2.temp.current": 33.0, + "gpu.3.compute.util": 0.0, + "gpu.3.mem.util": 0.0309326171875, + "gpu.3.mem.total": 40960.0, + "gpu.3.mem.used": 1267.0, + "gpu.3.mem.free": 39060.0, + "gpu.3.temp.current": 36.0, + "gpu.avg.compute.util": 0.0, + "gpu.avg.mem.util": 0.0309326171875, + "gpu.avg.mem.total": 40960.0, + "gpu.avg.mem.used": 1267.0, + "gpu.avg.mem.free": 39060.0, + "gpu.avg.temp.current": 34.75, + "os_platform": "Linux-6.1.0-44-cloud-amd64-x86_64-with-glibc2.31", + "python_version": "3.11.10", + "python_compiler": "GCC 9.4.0", + "git_branch": "main", + "git_commit_hash": "b21be29be0a1573fb4f78f849aea019cdb520862", + "cpu_model_name": "Intel(R) Xeon(R) CPU @ 2.20GHz", + "cpu_count": 24, + "gpu_model_name": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_driver": "550.90.12", + "rng_seed": 611108887 +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/ogbg_pytorch_09-13-2026-01-07-00.log b/logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/ogbg_pytorch_09-13-2026-01-07-00.log new file mode 100644 index 00000000..dd9decdb --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/ogbg_pytorch_09-13-2026-01-07-00.log @@ -0,0 +1,463 @@ +torchrun --redirects 1:0,2:0,3:0 --standalone --nnodes=1 --nproc_per_node=4 submission_runner.py --framework=pytorch --workload=ogbg --submission_path=submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py --data_dir=/data/ogbg --experiment_dir=/experiment_runs --experiment_name=submissions_a100/ademamix_golden/study_1 --overwrite=True --save_checkpoints=False --rng_seed=-1419473739 --torch_compile=true --tuning_ruleset=self 2>&1 | tee -a /logs/ogbg_pytorch_09-13-2026-01-07-00.log +W0913 01:07:26.394000 9 site-packages/torch/distributed/run.py:803] +W0913 01:07:26.394000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0913 01:07:26.394000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0913 01:07:26.394000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-13 01:07:40.906612: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-13 01:07:40.906612: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-13 01:07:40.906604: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-13 01:07:40.906605: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789261661.402858 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789261661.402861 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789261661.402899 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789261661.402885 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789261661.484140 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789261661.484156 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789261661.484158 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789261661.484169 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789261662.959637 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789261662.959647 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789261662.959642 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789261662.959654 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789261662.959684 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789261662.959684 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789261662.959684 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789261662.959685 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789261662.959687 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789261662.959687 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789261662.959688 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789261662.959688 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789261662.959689 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789261662.959690 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789261662.959690 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789261662.959691 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789261689.565051 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789261689.565045 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789261689.565082 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789261689.568920 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W913 01:08:19.009486971 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0913 01:08:22.045259 140058254623936 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/ogbg_pytorch. +I0913 01:08:22.045259 140452716922048 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/ogbg_pytorch. +I0913 01:08:22.045259 139913010533568 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/ogbg_pytorch. +I0913 01:08:22.045296 140414510847168 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/ogbg_pytorch. +I0913 01:08:22.330052 140414510847168 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/ogbg_pytorch/trial_1. +I0913 01:08:22.605943 140414510847168 submission_runner.py:242] Initializing dataset. +I0913 01:08:22.606117 140414510847168 submission_runner.py:251] Initializing model. +W0913 01:08:31.438484 140414510847168 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0913 01:08:31.438484 140452716922048 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0913 01:08:31.438483 140058254623936 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0913 01:08:31.438483 139913010533568 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +I0913 01:08:35.170305 140414510847168 submission_runner.py:294] Initializing optimizer. +I0913 01:08:35.170994 140414510847168 submission_runner.py:299] Initializing metrics bundle. +I0913 01:08:35.171153 140414510847168 submission_runner.py:321] Initializing checkpoint and logger. +I0913 01:08:35.173339 140414510847168 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_1/ogbg_pytorch/trial_1/meta_data_0.json. +I0913 01:08:35.173401 140452716922048 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0913 01:08:35.173421 139913010533568 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0913 01:08:35.173450 140058254623936 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0913 01:08:35.173548 140414510847168 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0913 01:08:35.173559 140452716922048 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0913 01:08:35.173572 139913010533568 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0913 01:08:35.173605 140414510847168 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0913 01:08:35.173624 140058254623936 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0913 01:08:35.658640 140414510847168 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_1/ogbg_pytorch/trial_1/flags_0.json. +I0913 01:08:35.734925 140414510847168 submission_runner.py:359] Starting training loop. +I0913 01:08:40.528908 140414510847168 dataset_info.py:707] Load dataset info from /data/ogbg/ogbg_molpcba/0.1.3 +I0913 01:08:40.583129 140414510847168 reader.py:262] Creating a tf.data.Dataset reading 8 files located in folders: /data/ogbg/ogbg_molpcba/0.1.3. +WARNING:tensorflow:From /usr/local/lib/python3.11/site-packages/tensorflow_datasets/core/reader.py:102: CounterV2 (from tensorflow.python.data.experimental.ops.counter) is deprecated and will be removed in a future version. +Instructions for updating: +Use `tf.data.Dataset.counter(...)` instead. +W0913 01:08:40.844721 140414510847168 deprecation.py:50] From /usr/local/lib/python3.11/site-packages/tensorflow_datasets/core/reader.py:102: CounterV2 (from tensorflow.python.data.experimental.ops.counter) is deprecated and will be removed in a future version. +Instructions for updating: +Use `tf.data.Dataset.counter(...)` instead. +I0913 01:08:41.307629 140414510847168 logging_logger.py:49] Constructing tf.data.Dataset ogbg_molpcba for split train, from /data/ogbg/ogbg_molpcba/0.1.3 +I0913 01:09:07.107853 140389126174464 logging_writer.py:48] [0] global_step=0, grad_norm=0.5, loss=0.728479 +I0913 01:09:07.298723 140414510847168 submission.py:307] 0) loss = 0.728, grad_norm = 0.500 +I0913 01:09:07.733063 140414510847168 spec.py:333] Evaluating on the training split. +I0913 01:09:07.736300 140414510847168 dataset_info.py:707] Load dataset info from /data/ogbg/ogbg_molpcba/0.1.3 +I0913 01:09:07.739694 140414510847168 reader.py:262] Creating a tf.data.Dataset reading 8 files located in folders: /data/ogbg/ogbg_molpcba/0.1.3. +I0913 01:09:07.801130 140414510847168 logging_logger.py:49] Constructing tf.data.Dataset ogbg_molpcba for split train, from /data/ogbg/ogbg_molpcba/0.1.3 +I0913 01:09:47.849410 140414510847168 spec.py:346] Evaluating on the validation split. +I0913 01:09:47.851808 140414510847168 dataset_info.py:707] Load dataset info from /data/ogbg/ogbg_molpcba/0.1.3 +I0913 01:09:47.855218 140414510847168 reader.py:262] Creating a tf.data.Dataset reading 1 files located in folders: /data/ogbg/ogbg_molpcba/0.1.3. +I0913 01:09:47.919281 140414510847168 logging_logger.py:49] Constructing tf.data.Dataset ogbg_molpcba for split validation, from /data/ogbg/ogbg_molpcba/0.1.3 +I0913 01:10:07.011831 140414510847168 spec.py:363] Evaluating on the test split. +I0913 01:10:07.014056 140414510847168 dataset_info.py:707] Load dataset info from /data/ogbg/ogbg_molpcba/0.1.3 +I0913 01:10:07.017430 140414510847168 reader.py:262] Creating a tf.data.Dataset reading 1 files located in folders: /data/ogbg/ogbg_molpcba/0.1.3. +I0913 01:10:07.076837 140414510847168 logging_logger.py:49] Constructing tf.data.Dataset ogbg_molpcba for split test, from /data/ogbg/ogbg_molpcba/0.1.3 +I0913 01:10:26.108055 140414510847168 submission_runner.py:516] Time since start: 110.37s, Step: 1, {'train/accuracy': 0.5586510674202247, 'train/loss': 0.7345021011061005, 'train/mean_average_precision': 0.023466601473233494, 'validation/accuracy': 0.5490050383355145, 'validation/loss': 0.7403665985795889, 'validation/mean_average_precision': 0.02634112715062556, 'validation/num_examples': 43793, 'test/accuracy': 0.5467212161999395, 'test/loss': 0.7414811802846774, 'test/mean_average_precision': 0.02823475507662738, 'test/num_examples': 43793, 'score': 31.565271377563477, 'total_duration': 110.37323498725891, 'accumulated_submission_time': 31.565271377563477, 'accumulated_eval_time': 78.37495589256287, 'accumulated_logging_time': 0} +I0913 01:10:26.127695 140379747440384 logging_writer.py:48] [1] accumulated_eval_time=78.375, accumulated_logging_time=0, accumulated_submission_time=31.5653, global_step=1, preemption_count=0, score=31.5653, test/accuracy=0.546721, test/loss=0.741481, test/mean_average_precision=0.0282348, test/num_examples=43793, total_duration=110.373, train/accuracy=0.558651, train/loss=0.734502, train/mean_average_precision=0.0234666, validation/accuracy=0.549005, validation/loss=0.740367, validation/mean_average_precision=0.0263411, validation/num_examples=43793 +I0913 01:10:26.834922 140379755833088 logging_writer.py:48] [1] global_step=1, grad_norm=0.5, loss=0.730535 +I0913 01:10:26.837943 140414510847168 submission.py:307] 1) loss = 0.731, grad_norm = 0.500 +I0913 01:10:27.084076 140379747440384 logging_writer.py:48] [2] global_step=2, grad_norm=0.5, loss=0.729496 +I0913 01:10:27.087171 140414510847168 submission.py:307] 2) loss = 0.729, grad_norm = 0.500 +I0913 01:10:27.329640 140379755833088 logging_writer.py:48] [3] global_step=3, grad_norm=0.5, loss=0.728626 +I0913 01:10:27.332563 140414510847168 submission.py:307] 3) loss = 0.729, grad_norm = 0.500 +I0913 01:10:27.576979 140379747440384 logging_writer.py:48] [4] global_step=4, grad_norm=0.5, loss=0.728981 +I0913 01:10:27.579923 140414510847168 submission.py:307] 4) loss = 0.729, grad_norm = 0.500 +I0913 01:10:27.819232 140379755833088 logging_writer.py:48] [5] global_step=5, grad_norm=0.5, loss=0.725472 +I0913 01:10:27.822232 140414510847168 submission.py:307] 5) loss = 0.725, grad_norm = 0.500 +I0913 01:10:28.071012 140379747440384 logging_writer.py:48] [6] global_step=6, grad_norm=0.5, loss=0.725856 +I0913 01:10:28.074082 140414510847168 submission.py:307] 6) loss = 0.726, grad_norm = 0.500 +I0913 01:10:28.328935 140379755833088 logging_writer.py:48] [7] global_step=7, grad_norm=0.5, loss=0.721924 +I0913 01:10:28.331948 140414510847168 submission.py:307] 7) loss = 0.722, grad_norm = 0.500 +I0913 01:10:28.575167 140379747440384 logging_writer.py:48] [8] global_step=8, grad_norm=0.5, loss=0.719258 +I0913 01:10:28.578378 140414510847168 submission.py:307] 8) loss = 0.719, grad_norm = 0.500 +I0913 01:10:28.820628 140379755833088 logging_writer.py:48] [9] global_step=9, grad_norm=0.5, loss=0.711758 +I0913 01:10:28.823832 140414510847168 submission.py:307] 9) loss = 0.712, grad_norm = 0.500 +I0913 01:10:29.066415 140379747440384 logging_writer.py:48] [10] global_step=10, grad_norm=0.5, loss=0.71116 +I0913 01:10:29.069728 140414510847168 submission.py:307] 10) loss = 0.711, grad_norm = 0.500 +I0913 01:10:29.313488 140379755833088 logging_writer.py:48] [11] global_step=11, grad_norm=0.5, loss=0.70956 +I0913 01:10:29.317430 140414510847168 submission.py:307] 11) loss = 0.710, grad_norm = 0.500 +I0913 01:10:29.561283 140379747440384 logging_writer.py:48] [12] global_step=12, grad_norm=0.5, loss=0.70263 +I0913 01:10:29.564654 140414510847168 submission.py:307] 12) loss = 0.703, grad_norm = 0.500 +I0913 01:10:29.809994 140379755833088 logging_writer.py:48] [13] global_step=13, grad_norm=0.5, loss=0.696168 +I0913 01:10:29.813163 140414510847168 submission.py:307] 13) loss = 0.696, grad_norm = 0.500 +I0913 01:10:30.053087 140379747440384 logging_writer.py:48] [14] global_step=14, grad_norm=0.5, loss=0.690788 +I0913 01:10:30.056051 140414510847168 submission.py:307] 14) loss = 0.691, grad_norm = 0.500 +I0913 01:10:30.297043 140379755833088 logging_writer.py:48] [15] global_step=15, grad_norm=0.5, loss=0.684906 +I0913 01:10:30.300065 140414510847168 submission.py:307] 15) loss = 0.685, grad_norm = 0.500 +I0913 01:10:30.542372 140379747440384 logging_writer.py:48] [16] global_step=16, grad_norm=0.5, loss=0.678718 +I0913 01:10:30.545261 140414510847168 submission.py:307] 16) loss = 0.679, grad_norm = 0.500 +I0913 01:10:30.786581 140379755833088 logging_writer.py:48] [17] global_step=17, grad_norm=0.5, loss=0.671283 +I0913 01:10:30.789614 140414510847168 submission.py:307] 17) loss = 0.671, grad_norm = 0.500 +I0913 01:10:31.030599 140379747440384 logging_writer.py:48] [18] global_step=18, grad_norm=0.5, loss=0.669953 +I0913 01:10:31.033588 140414510847168 submission.py:307] 18) loss = 0.670, grad_norm = 0.500 +I0913 01:10:31.274223 140379755833088 logging_writer.py:48] [19] global_step=19, grad_norm=0.5, loss=0.663981 +I0913 01:10:31.277163 140414510847168 submission.py:307] 19) loss = 0.664, grad_norm = 0.500 +I0913 01:10:31.520115 140379747440384 logging_writer.py:48] [20] global_step=20, grad_norm=0.5, loss=0.652072 +I0913 01:10:31.523119 140414510847168 submission.py:307] 20) loss = 0.652, grad_norm = 0.500 +I0913 01:10:31.763725 140379755833088 logging_writer.py:48] [21] global_step=21, grad_norm=0.5, loss=0.645859 +I0913 01:10:31.767051 140414510847168 submission.py:307] 21) loss = 0.646, grad_norm = 0.500 +I0913 01:10:32.011507 140379747440384 logging_writer.py:48] [22] global_step=22, grad_norm=0.5, loss=0.637613 +I0913 01:10:32.014412 140414510847168 submission.py:307] 22) loss = 0.638, grad_norm = 0.500 +I0913 01:10:32.253799 140379755833088 logging_writer.py:48] [23] global_step=23, grad_norm=0.5, loss=0.629899 +I0913 01:10:32.256798 140414510847168 submission.py:307] 23) loss = 0.630, grad_norm = 0.500 +I0913 01:10:32.501950 140379747440384 logging_writer.py:48] [24] global_step=24, grad_norm=0.5, loss=0.627101 +I0913 01:10:32.504969 140414510847168 submission.py:307] 24) loss = 0.627, grad_norm = 0.500 +I0913 01:10:32.748251 140379755833088 logging_writer.py:48] [25] global_step=25, grad_norm=0.5, loss=0.615109 +I0913 01:10:32.751224 140414510847168 submission.py:307] 25) loss = 0.615, grad_norm = 0.500 +I0913 01:10:32.995712 140379747440384 logging_writer.py:48] [26] global_step=26, grad_norm=0.5, loss=0.604968 +I0913 01:10:32.998743 140414510847168 submission.py:307] 26) loss = 0.605, grad_norm = 0.500 +I0913 01:10:33.243538 140379755833088 logging_writer.py:48] [27] global_step=27, grad_norm=0.5, loss=0.598412 +I0913 01:10:33.246548 140414510847168 submission.py:307] 27) loss = 0.598, grad_norm = 0.500 +I0913 01:10:33.489198 140379747440384 logging_writer.py:48] [28] global_step=28, grad_norm=0.5, loss=0.590896 +I0913 01:10:33.492168 140414510847168 submission.py:307] 28) loss = 0.591, grad_norm = 0.500 +I0913 01:10:33.732647 140379755833088 logging_writer.py:48] [29] global_step=29, grad_norm=0.5, loss=0.580914 +I0913 01:10:33.735737 140414510847168 submission.py:307] 29) loss = 0.581, grad_norm = 0.500 +I0913 01:10:33.978316 140379747440384 logging_writer.py:48] [30] global_step=30, grad_norm=0.5, loss=0.570592 +I0913 01:10:33.981392 140414510847168 submission.py:307] 30) loss = 0.571, grad_norm = 0.500 +I0913 01:10:34.228562 140379755833088 logging_writer.py:48] [31] global_step=31, grad_norm=0.5, loss=0.564664 +I0913 01:10:34.231541 140414510847168 submission.py:307] 31) loss = 0.565, grad_norm = 0.500 +I0913 01:10:34.480916 140379747440384 logging_writer.py:48] [32] global_step=32, grad_norm=0.5, loss=0.557313 +I0913 01:10:34.484096 140414510847168 submission.py:307] 32) loss = 0.557, grad_norm = 0.500 +I0913 01:10:34.731414 140379755833088 logging_writer.py:48] [33] global_step=33, grad_norm=0.5, loss=0.547977 +I0913 01:10:34.734467 140414510847168 submission.py:307] 33) loss = 0.548, grad_norm = 0.500 +I0913 01:10:34.979351 140379747440384 logging_writer.py:48] [34] global_step=34, grad_norm=0.5, loss=0.540141 +I0913 01:10:34.982441 140414510847168 submission.py:307] 34) loss = 0.540, grad_norm = 0.500 +I0913 01:10:35.232241 140379755833088 logging_writer.py:48] [35] global_step=35, grad_norm=0.499999, loss=0.531826 +I0913 01:10:35.235276 140414510847168 submission.py:307] 35) loss = 0.532, grad_norm = 0.500 +I0913 01:10:35.480947 140379747440384 logging_writer.py:48] [36] global_step=36, grad_norm=0.499999, loss=0.523864 +I0913 01:10:35.484035 140414510847168 submission.py:307] 36) loss = 0.524, grad_norm = 0.500 +I0913 01:10:35.730905 140379755833088 logging_writer.py:48] [37] global_step=37, grad_norm=0.499999, loss=0.513047 +I0913 01:10:35.736908 140414510847168 submission.py:307] 37) loss = 0.513, grad_norm = 0.500 +I0913 01:10:35.983936 140379747440384 logging_writer.py:48] [38] global_step=38, grad_norm=0.499999, loss=0.509439 +I0913 01:10:35.987594 140414510847168 submission.py:307] 38) loss = 0.509, grad_norm = 0.500 +I0913 01:10:36.232926 140379755833088 logging_writer.py:48] [39] global_step=39, grad_norm=0.499999, loss=0.500375 +I0913 01:10:36.236407 140414510847168 submission.py:307] 39) loss = 0.500, grad_norm = 0.500 +I0913 01:10:36.480942 140379747440384 logging_writer.py:48] [40] global_step=40, grad_norm=0.499999, loss=0.49546 +I0913 01:10:36.484062 140414510847168 submission.py:307] 40) loss = 0.495, grad_norm = 0.500 +I0913 01:10:36.730657 140379755833088 logging_writer.py:48] [41] global_step=41, grad_norm=0.499999, loss=0.49039 +I0913 01:10:36.734226 140414510847168 submission.py:307] 41) loss = 0.490, grad_norm = 0.500 +I0913 01:10:36.977590 140379747440384 logging_writer.py:48] [42] global_step=42, grad_norm=0.499999, loss=0.483528 +I0913 01:10:36.981069 140414510847168 submission.py:307] 42) loss = 0.484, grad_norm = 0.500 +I0913 01:10:37.222315 140379755833088 logging_writer.py:48] [43] global_step=43, grad_norm=0.499999, loss=0.481444 +I0913 01:10:37.225395 140414510847168 submission.py:307] 43) loss = 0.481, grad_norm = 0.500 +I0913 01:10:37.468041 140379747440384 logging_writer.py:48] [44] global_step=44, grad_norm=0.499999, loss=0.471582 +I0913 01:10:37.473731 140414510847168 submission.py:307] 44) loss = 0.472, grad_norm = 0.500 +I0913 01:10:37.715537 140379755833088 logging_writer.py:48] [45] global_step=45, grad_norm=0.499999, loss=0.468898 +I0913 01:10:37.719584 140414510847168 submission.py:307] 45) loss = 0.469, grad_norm = 0.500 +I0913 01:10:37.964204 140379747440384 logging_writer.py:48] [46] global_step=46, grad_norm=0.499999, loss=0.464524 +I0913 01:10:37.967385 140414510847168 submission.py:307] 46) loss = 0.465, grad_norm = 0.500 +I0913 01:10:38.212105 140379755833088 logging_writer.py:48] [47] global_step=47, grad_norm=0.499999, loss=0.45727 +I0913 01:10:38.215238 140414510847168 submission.py:307] 47) loss = 0.457, grad_norm = 0.500 +I0913 01:10:38.460186 140379747440384 logging_writer.py:48] [48] global_step=48, grad_norm=0.499999, loss=0.45304 +I0913 01:10:38.463624 140414510847168 submission.py:307] 48) loss = 0.453, grad_norm = 0.500 +I0913 01:10:38.708217 140379755833088 logging_writer.py:48] [49] global_step=49, grad_norm=0.499999, loss=0.452006 +I0913 01:10:38.711167 140414510847168 submission.py:307] 49) loss = 0.452, grad_norm = 0.500 +I0913 01:10:38.957215 140379747440384 logging_writer.py:48] [50] global_step=50, grad_norm=0.499999, loss=0.44577 +I0913 01:10:38.960225 140414510847168 submission.py:307] 50) loss = 0.446, grad_norm = 0.500 +I0913 01:10:39.203263 140379755833088 logging_writer.py:48] [51] global_step=51, grad_norm=0.499999, loss=0.440786 +I0913 01:10:39.206207 140414510847168 submission.py:307] 51) loss = 0.441, grad_norm = 0.500 +I0913 01:10:39.449652 140379747440384 logging_writer.py:48] [52] global_step=52, grad_norm=0.499999, loss=0.43798 +I0913 01:10:39.452837 140414510847168 submission.py:307] 52) loss = 0.438, grad_norm = 0.500 +I0913 01:10:39.697722 140379755833088 logging_writer.py:48] [53] global_step=53, grad_norm=0.497673, loss=0.440495 +I0913 01:10:39.700728 140414510847168 submission.py:307] 53) loss = 0.440, grad_norm = 0.498 +I0913 01:10:39.943007 140379747440384 logging_writer.py:48] [54] global_step=54, grad_norm=0.494459, loss=0.431466 +I0913 01:10:39.945978 140414510847168 submission.py:307] 54) loss = 0.431, grad_norm = 0.494 +I0913 01:10:40.189751 140379755833088 logging_writer.py:48] [55] global_step=55, grad_norm=0.486141, loss=0.427144 +I0913 01:10:40.192765 140414510847168 submission.py:307] 55) loss = 0.427, grad_norm = 0.486 +I0913 01:10:40.433347 140379747440384 logging_writer.py:48] [56] global_step=56, grad_norm=0.480954, loss=0.423342 +I0913 01:10:40.436396 140414510847168 submission.py:307] 56) loss = 0.423, grad_norm = 0.481 +I0913 01:10:40.678837 140379755833088 logging_writer.py:48] [57] global_step=57, grad_norm=0.471848, loss=0.425899 +I0913 01:10:40.681859 140414510847168 submission.py:307] 57) loss = 0.426, grad_norm = 0.472 +I0913 01:10:40.925402 140379747440384 logging_writer.py:48] [58] global_step=58, grad_norm=0.467501, loss=0.420367 +I0913 01:10:40.928363 140414510847168 submission.py:307] 58) loss = 0.420, grad_norm = 0.468 +I0913 01:10:41.173760 140379755833088 logging_writer.py:48] [59] global_step=59, grad_norm=0.463369, loss=0.419721 +I0913 01:10:41.176776 140414510847168 submission.py:307] 59) loss = 0.420, grad_norm = 0.463 +I0913 01:10:41.421303 140379747440384 logging_writer.py:48] [60] global_step=60, grad_norm=0.461335, loss=0.413886 +I0913 01:10:41.424297 140414510847168 submission.py:307] 60) loss = 0.414, grad_norm = 0.461 +I0913 01:10:41.666218 140379755833088 logging_writer.py:48] [61] global_step=61, grad_norm=0.45947, loss=0.411149 +I0913 01:10:41.669307 140414510847168 submission.py:307] 61) loss = 0.411, grad_norm = 0.459 +I0913 01:10:41.912588 140379747440384 logging_writer.py:48] [62] global_step=62, grad_norm=0.459573, loss=0.406947 +I0913 01:10:41.915648 140414510847168 submission.py:307] 62) loss = 0.407, grad_norm = 0.460 +I0913 01:10:42.159775 140379755833088 logging_writer.py:48] [63] global_step=63, grad_norm=0.455153, loss=0.406738 +I0913 01:10:42.162786 140414510847168 submission.py:307] 63) loss = 0.407, grad_norm = 0.455 +I0913 01:10:42.409357 140379747440384 logging_writer.py:48] [64] global_step=64, grad_norm=0.449655, loss=0.403712 +I0913 01:10:42.412341 140414510847168 submission.py:307] 64) loss = 0.404, grad_norm = 0.450 +I0913 01:10:42.659032 140379755833088 logging_writer.py:48] [65] global_step=65, grad_norm=0.447662, loss=0.402483 +I0913 01:10:42.662128 140414510847168 submission.py:307] 65) loss = 0.402, grad_norm = 0.448 +I0913 01:10:42.907837 140379747440384 logging_writer.py:48] [66] global_step=66, grad_norm=0.443879, loss=0.398607 +I0913 01:10:42.910835 140414510847168 submission.py:307] 66) loss = 0.399, grad_norm = 0.444 +I0913 01:10:43.152867 140379755833088 logging_writer.py:48] [67] global_step=67, grad_norm=0.433898, loss=0.399821 +I0913 01:10:43.155859 140414510847168 submission.py:307] 67) loss = 0.400, grad_norm = 0.434 +I0913 01:10:43.400990 140379747440384 logging_writer.py:48] [68] global_step=68, grad_norm=0.43545, loss=0.392929 +I0913 01:10:43.404043 140414510847168 submission.py:307] 68) loss = 0.393, grad_norm = 0.435 +I0913 01:10:43.649129 140379755833088 logging_writer.py:48] [69] global_step=69, grad_norm=0.43277, loss=0.393344 +I0913 01:10:43.652137 140414510847168 submission.py:307] 69) loss = 0.393, grad_norm = 0.433 +I0913 01:10:43.896124 140379747440384 logging_writer.py:48] [70] global_step=70, grad_norm=0.433609, loss=0.391122 +I0913 01:10:43.899049 140414510847168 submission.py:307] 70) loss = 0.391, grad_norm = 0.434 +I0913 01:10:44.143917 140379755833088 logging_writer.py:48] [71] global_step=71, grad_norm=0.452115, loss=0.387052 +I0913 01:10:44.146940 140414510847168 submission.py:307] 71) loss = 0.387, grad_norm = 0.452 +I0913 01:10:44.390094 140379747440384 logging_writer.py:48] [72] global_step=72, grad_norm=0.44172, loss=0.38507 +I0913 01:10:44.393000 140414510847168 submission.py:307] 72) loss = 0.385, grad_norm = 0.442 +I0913 01:10:44.635804 140379755833088 logging_writer.py:48] [73] global_step=73, grad_norm=0.42957, loss=0.386344 +I0913 01:10:44.638799 140414510847168 submission.py:307] 73) loss = 0.386, grad_norm = 0.430 +I0913 01:10:44.886027 140379747440384 logging_writer.py:48] [74] global_step=74, grad_norm=0.424712, loss=0.382086 +I0913 01:10:44.889068 140414510847168 submission.py:307] 74) loss = 0.382, grad_norm = 0.425 +I0913 01:10:45.134810 140379755833088 logging_writer.py:48] [75] global_step=75, grad_norm=0.419183, loss=0.379433 +I0913 01:10:45.137793 140414510847168 submission.py:307] 75) loss = 0.379, grad_norm = 0.419 +I0913 01:10:45.387621 140379747440384 logging_writer.py:48] [76] global_step=76, grad_norm=0.417905, loss=0.377403 +I0913 01:10:45.390755 140414510847168 submission.py:307] 76) loss = 0.377, grad_norm = 0.418 +I0913 01:10:45.639945 140379755833088 logging_writer.py:48] [77] global_step=77, grad_norm=0.41146, loss=0.376207 +I0913 01:10:45.642960 140414510847168 submission.py:307] 77) loss = 0.376, grad_norm = 0.411 +I0913 01:10:45.894589 140379747440384 logging_writer.py:48] [78] global_step=78, grad_norm=0.408512, loss=0.371575 +I0913 01:10:45.897725 140414510847168 submission.py:307] 78) loss = 0.372, grad_norm = 0.409 +I0913 01:10:46.146412 140379755833088 logging_writer.py:48] [79] global_step=79, grad_norm=0.403855, loss=0.370022 +I0913 01:10:46.149406 140414510847168 submission.py:307] 79) loss = 0.370, grad_norm = 0.404 +I0913 01:10:46.397042 140379747440384 logging_writer.py:48] [80] global_step=80, grad_norm=0.403979, loss=0.368339 +I0913 01:10:46.400035 140414510847168 submission.py:307] 80) loss = 0.368, grad_norm = 0.404 +I0913 01:10:46.645499 140379755833088 logging_writer.py:48] [81] global_step=81, grad_norm=0.400467, loss=0.36676 +I0913 01:10:46.648409 140414510847168 submission.py:307] 81) loss = 0.367, grad_norm = 0.400 +I0913 01:10:46.893962 140379747440384 logging_writer.py:48] [82] global_step=82, grad_norm=0.398876, loss=0.368283 +I0913 01:10:46.896877 140414510847168 submission.py:307] 82) loss = 0.368, grad_norm = 0.399 +I0913 01:10:47.140259 140379755833088 logging_writer.py:48] [83] global_step=83, grad_norm=0.397733, loss=0.363378 +I0913 01:10:47.143313 140414510847168 submission.py:307] 83) loss = 0.363, grad_norm = 0.398 +I0913 01:10:47.391342 140379747440384 logging_writer.py:48] [84] global_step=84, grad_norm=0.393243, loss=0.361512 +I0913 01:10:47.394338 140414510847168 submission.py:307] 84) loss = 0.362, grad_norm = 0.393 +I0913 01:10:47.642026 140379755833088 logging_writer.py:48] [85] global_step=85, grad_norm=0.388189, loss=0.36242 +I0913 01:10:47.645005 140414510847168 submission.py:307] 85) loss = 0.362, grad_norm = 0.388 +I0913 01:10:47.893253 140379747440384 logging_writer.py:48] [86] global_step=86, grad_norm=0.386464, loss=0.360579 +I0913 01:10:47.896248 140414510847168 submission.py:307] 86) loss = 0.361, grad_norm = 0.386 +I0913 01:10:48.143593 140379755833088 logging_writer.py:48] [87] global_step=87, grad_norm=0.391082, loss=0.356344 +I0913 01:10:48.146655 140414510847168 submission.py:307] 87) loss = 0.356, grad_norm = 0.391 +I0913 01:10:48.394699 140379747440384 logging_writer.py:48] [88] global_step=88, grad_norm=0.386468, loss=0.355973 +I0913 01:10:48.397742 140414510847168 submission.py:307] 88) loss = 0.356, grad_norm = 0.386 +I0913 01:10:48.642966 140379755833088 logging_writer.py:48] [89] global_step=89, grad_norm=0.382687, loss=0.356793 +I0913 01:10:48.646021 140414510847168 submission.py:307] 89) loss = 0.357, grad_norm = 0.383 +I0913 01:10:48.891215 140379747440384 logging_writer.py:48] [90] global_step=90, grad_norm=0.385518, loss=0.352109 +I0913 01:10:48.894249 140414510847168 submission.py:307] 90) loss = 0.352, grad_norm = 0.386 +I0913 01:10:49.142618 140379755833088 logging_writer.py:48] [91] global_step=91, grad_norm=0.382113, loss=0.351625 +I0913 01:10:49.145693 140414510847168 submission.py:307] 91) loss = 0.352, grad_norm = 0.382 +I0913 01:10:49.391429 140379747440384 logging_writer.py:48] [92] global_step=92, grad_norm=0.380693, loss=0.351116 +I0913 01:10:49.394434 140414510847168 submission.py:307] 92) loss = 0.351, grad_norm = 0.381 +I0913 01:10:49.638231 140379755833088 logging_writer.py:48] [93] global_step=93, grad_norm=0.380173, loss=0.348481 +I0913 01:10:49.641278 140414510847168 submission.py:307] 93) loss = 0.348, grad_norm = 0.380 +I0913 01:10:49.887771 140379747440384 logging_writer.py:48] [94] global_step=94, grad_norm=0.379323, loss=0.346351 +I0913 01:10:49.890995 140414510847168 submission.py:307] 94) loss = 0.346, grad_norm = 0.379 +I0913 01:10:50.137620 140379755833088 logging_writer.py:48] [95] global_step=95, grad_norm=0.375825, loss=0.344865 +I0913 01:10:50.140614 140414510847168 submission.py:307] 95) loss = 0.345, grad_norm = 0.376 +I0913 01:10:50.389481 140379747440384 logging_writer.py:48] [96] global_step=96, grad_norm=0.375272, loss=0.342934 +I0913 01:10:50.392537 140414510847168 submission.py:307] 96) loss = 0.343, grad_norm = 0.375 +I0913 01:10:50.636295 140379755833088 logging_writer.py:48] [97] global_step=97, grad_norm=0.377983, loss=0.33952 +I0913 01:10:50.639450 140414510847168 submission.py:307] 97) loss = 0.340, grad_norm = 0.378 +I0913 01:10:50.883380 140379747440384 logging_writer.py:48] [98] global_step=98, grad_norm=0.372457, loss=0.338567 +I0913 01:10:50.886449 140414510847168 submission.py:307] 98) loss = 0.339, grad_norm = 0.372 +I0913 01:10:51.130818 140379755833088 logging_writer.py:48] [99] global_step=99, grad_norm=0.372528, loss=0.340341 +I0913 01:10:51.133859 140414510847168 submission.py:307] 99) loss = 0.340, grad_norm = 0.373 +I0913 01:10:51.379771 140379747440384 logging_writer.py:48] [100] global_step=100, grad_norm=0.369444, loss=0.339648 +I0913 01:10:51.382823 140414510847168 submission.py:307] 100) loss = 0.340, grad_norm = 0.369 +I0913 01:12:26.944788 140379755833088 logging_writer.py:48] [500] global_step=500, grad_norm=0.030448, loss=0.0682298 +I0913 01:12:26.948212 140414510847168 submission.py:307] 500) loss = 0.068, grad_norm = 0.030 +I0913 01:14:24.499054 140379747440384 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.0193225, loss=0.0504973 +I0913 01:14:24.502445 140414510847168 submission.py:307] 1000) loss = 0.050, grad_norm = 0.019 +I0913 01:16:22.543859 140379755833088 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.0229361, loss=0.0448496 +I0913 01:16:22.547434 140414510847168 submission.py:307] 1500) loss = 0.045, grad_norm = 0.023 +I0913 01:17:58.593968 140414510847168 spec.py:333] Evaluating on the training split. +I0913 01:18:24.894428 140414510847168 spec.py:346] Evaluating on the validation split. +I0913 01:18:26.907112 140414510847168 spec.py:363] Evaluating on the test split. +I0913 01:18:28.986604 140414510847168 submission_runner.py:516] Time since start: 593.25s, Step: 1909, {'train/accuracy': 0.987392573965418, 'train/loss': 0.04626145357260643, 'train/mean_average_precision': 0.10833063408675046, 'validation/accuracy': 0.9847544887136738, 'validation/loss': 0.055170290169075695, 'validation/mean_average_precision': 0.10860620174572136, 'validation/num_examples': 43793, 'test/accuracy': 0.9837648527381597, 'test/loss': 0.05830736797729859, 'test/mean_average_precision': 0.10742968659137667, 'test/num_examples': 43793, 'score': 482.64182305336, 'total_duration': 593.2517759799957, 'accumulated_submission_time': 482.64182305336, 'accumulated_eval_time': 108.76751208305359, 'accumulated_logging_time': 0.029638051986694336} +I0913 01:18:29.010369 140379747440384 logging_writer.py:48] [1909] accumulated_eval_time=108.768, accumulated_logging_time=0.0296381, accumulated_submission_time=482.642, global_step=1909, preemption_count=0, score=482.642, test/accuracy=0.983765, test/loss=0.0583074, test/mean_average_precision=0.10743, test/num_examples=43793, total_duration=593.252, train/accuracy=0.987393, train/loss=0.0462615, train/mean_average_precision=0.108331, validation/accuracy=0.984754, validation/loss=0.0551703, validation/mean_average_precision=0.108606, validation/num_examples=43793 +I0913 01:18:51.417424 140379755833088 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.0189821, loss=0.0459201 +I0913 01:18:51.420550 140414510847168 submission.py:307] 2000) loss = 0.046, grad_norm = 0.019 +I0913 01:20:49.048781 140379747440384 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.0437944, loss=0.0394203 +I0913 01:20:49.052136 140414510847168 submission.py:307] 2500) loss = 0.039, grad_norm = 0.044 +I0913 01:22:47.360907 140379755833088 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.0187581, loss=0.0440383 +I0913 01:22:47.364858 140414510847168 submission.py:307] 3000) loss = 0.044, grad_norm = 0.019 +I0913 01:24:45.775686 140379747440384 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.0269089, loss=0.0456432 +I0913 01:24:45.779088 140414510847168 submission.py:307] 3500) loss = 0.046, grad_norm = 0.027 +I0913 01:26:01.485800 140414510847168 spec.py:333] Evaluating on the training split. +I0913 01:26:27.628592 140414510847168 spec.py:346] Evaluating on the validation split. +I0913 01:26:29.643171 140414510847168 spec.py:363] Evaluating on the test split. +I0913 01:26:31.709884 140414510847168 submission_runner.py:516] Time since start: 1075.97s, Step: 3823, {'train/accuracy': 0.9884848246424989, 'train/loss': 0.040407567271189444, 'train/mean_average_precision': 0.19120728049328006, 'validation/accuracy': 0.9854151309187859, 'validation/loss': 0.05075225176495994, 'validation/mean_average_precision': 0.16287674577030156, 'validation/num_examples': 43793, 'test/accuracy': 0.984448371389417, 'test/loss': 0.05355386926094914, 'test/mean_average_precision': 0.1640744205277744, 'test/num_examples': 43793, 'score': 933.7262935638428, 'total_duration': 1075.9749946594238, 'accumulated_submission_time': 933.7262935638428, 'accumulated_eval_time': 138.99151515960693, 'accumulated_logging_time': 0.0630345344543457} +I0913 01:26:31.733505 140379755833088 logging_writer.py:48] [3823] accumulated_eval_time=138.992, accumulated_logging_time=0.0630345, accumulated_submission_time=933.726, global_step=3823, preemption_count=0, score=933.726, test/accuracy=0.984448, test/loss=0.0535539, test/mean_average_precision=0.164074, test/num_examples=43793, total_duration=1075.97, train/accuracy=0.988485, train/loss=0.0404076, train/mean_average_precision=0.191207, validation/accuracy=0.985415, validation/loss=0.0507523, validation/mean_average_precision=0.162877, validation/num_examples=43793 +I0913 01:27:14.484435 140379747440384 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.026149, loss=0.0436869 +I0913 01:27:14.487650 140414510847168 submission.py:307] 4000) loss = 0.044, grad_norm = 0.026 +I0913 01:29:13.940930 140379755833088 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.0243712, loss=0.0486334 +I0913 01:29:13.944608 140414510847168 submission.py:307] 4500) loss = 0.049, grad_norm = 0.024 +I0913 01:31:13.586194 140379747440384 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.0201098, loss=0.0436581 +I0913 01:31:13.589648 140414510847168 submission.py:307] 5000) loss = 0.044, grad_norm = 0.020 +I0913 01:33:11.810143 140379755833088 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.0308451, loss=0.0378625 +I0913 01:33:11.813530 140414510847168 submission.py:307] 5500) loss = 0.038, grad_norm = 0.031 +I0913 01:34:04.318002 140414510847168 spec.py:333] Evaluating on the training split. +I0913 01:34:30.704740 140414510847168 spec.py:346] Evaluating on the validation split. +I0913 01:34:32.730834 140414510847168 spec.py:363] Evaluating on the test split. +I0913 01:34:34.819263 140414510847168 submission_runner.py:516] Time since start: 1559.08s, Step: 5720, {'train/accuracy': 0.9886374956736488, 'train/loss': 0.03856164425364093, 'train/mean_average_precision': 0.23909901151243002, 'validation/accuracy': 0.9857841163347286, 'validation/loss': 0.048169271399936354, 'validation/mean_average_precision': 0.2052480629121319, 'validation/num_examples': 43793, 'test/accuracy': 0.9848665685679127, 'test/loss': 0.05095236984718325, 'test/mean_average_precision': 0.20260702704146777, 'test/num_examples': 43793, 'score': 1384.9259967803955, 'total_duration': 1559.084408044815, 'accumulated_submission_time': 1384.9259967803955, 'accumulated_eval_time': 169.49265098571777, 'accumulated_logging_time': 0.09615492820739746} +I0913 01:34:34.842361 140379747440384 logging_writer.py:48] [5720] accumulated_eval_time=169.493, accumulated_logging_time=0.0961549, accumulated_submission_time=1384.93, global_step=5720, preemption_count=0, score=1384.93, test/accuracy=0.984867, test/loss=0.0509524, test/mean_average_precision=0.202607, test/num_examples=43793, total_duration=1559.08, train/accuracy=0.988637, train/loss=0.0385616, train/mean_average_precision=0.239099, validation/accuracy=0.985784, validation/loss=0.0481693, validation/mean_average_precision=0.205248, validation/num_examples=43793 +I0913 01:35:42.652616 140379755833088 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.0200517, loss=0.0395983 +I0913 01:35:42.655851 140414510847168 submission.py:307] 6000) loss = 0.040, grad_norm = 0.020 +I0913 01:37:41.908269 140379747440384 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.0272362, loss=0.0385592 +I0913 01:37:41.912413 140414510847168 submission.py:307] 6500) loss = 0.039, grad_norm = 0.027 +I0913 01:39:41.417582 140379755833088 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.021751, loss=0.03493 +I0913 01:39:41.421043 140414510847168 submission.py:307] 7000) loss = 0.035, grad_norm = 0.022 +I0913 01:41:40.842155 140379747440384 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.0156814, loss=0.0324344 +I0913 01:41:40.845720 140414510847168 submission.py:307] 7500) loss = 0.032, grad_norm = 0.016 +I0913 01:42:07.385359 140414510847168 spec.py:333] Evaluating on the training split. +I0913 01:42:33.604250 140414510847168 spec.py:346] Evaluating on the validation split. +I0913 01:42:35.608260 140414510847168 spec.py:363] Evaluating on the test split. +I0913 01:42:37.662148 140414510847168 submission_runner.py:516] Time since start: 2041.93s, Step: 7610, {'train/accuracy': 0.9895910571669182, 'train/loss': 0.03510912069093631, 'train/mean_average_precision': 0.2911635532020825, 'validation/accuracy': 0.9861514780524714, 'validation/loss': 0.04606302087730849, 'validation/mean_average_precision': 0.22996665595578003, 'validation/num_examples': 43793, 'test/accuracy': 0.9853134036196587, 'test/loss': 0.048619133342149304, 'test/mean_average_precision': 0.2286057515900184, 'test/num_examples': 43793, 'score': 1836.0931000709534, 'total_duration': 2041.9273238182068, 'accumulated_submission_time': 1836.0931000709534, 'accumulated_eval_time': 199.7694013118744, 'accumulated_logging_time': 0.1288614273071289} +I0913 01:42:37.684312 140379755833088 logging_writer.py:48] [7610] accumulated_eval_time=199.769, accumulated_logging_time=0.128861, accumulated_submission_time=1836.09, global_step=7610, preemption_count=0, score=1836.09, test/accuracy=0.985313, test/loss=0.0486191, test/mean_average_precision=0.228606, test/num_examples=43793, total_duration=2041.93, train/accuracy=0.989591, train/loss=0.0351091, train/mean_average_precision=0.291164, validation/accuracy=0.986151, validation/loss=0.046063, validation/mean_average_precision=0.229967, validation/num_examples=43793 +I0913 01:44:11.389155 140379747440384 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.0163229, loss=0.0340497 +I0913 01:44:11.392511 140414510847168 submission.py:307] 8000) loss = 0.034, grad_norm = 0.016 +I0913 01:46:10.877550 140379755833088 logging_writer.py:48] [8500] global_step=8500, grad_norm=0.0200388, loss=0.0392392 +I0913 01:46:10.881026 140414510847168 submission.py:307] 8500) loss = 0.039, grad_norm = 0.020 +I0913 01:48:10.317507 140379747440384 logging_writer.py:48] [9000] global_step=9000, grad_norm=0.0169167, loss=0.0355976 +I0913 01:48:10.320794 140414510847168 submission.py:307] 9000) loss = 0.036, grad_norm = 0.017 +I0913 01:50:09.311642 140379755833088 logging_writer.py:48] [9500] global_step=9500, grad_norm=0.0149519, loss=0.0346768 +I0913 01:50:09.315127 140414510847168 submission.py:307] 9500) loss = 0.035, grad_norm = 0.015 +I0913 01:50:10.229608 140414510847168 spec.py:333] Evaluating on the training split. +I0913 01:50:36.603995 140414510847168 spec.py:346] Evaluating on the validation split. +I0913 01:50:38.625994 140414510847168 spec.py:363] Evaluating on the test split. +I0913 01:50:40.716366 140414510847168 submission_runner.py:516] Time since start: 2524.98s, Step: 9503, {'train/accuracy': 0.9898969062830736, 'train/loss': 0.03370176837601168, 'train/mean_average_precision': 0.3318149727723656, 'validation/accuracy': 0.9865431952432138, 'validation/loss': 0.04514741682808933, 'validation/mean_average_precision': 0.24401900095267037, 'validation/num_examples': 43793, 'test/accuracy': 0.9856915920046427, 'test/loss': 0.04799593057926837, 'test/mean_average_precision': 0.23806589574507442, 'test/num_examples': 43793, 'score': 2287.257420063019, 'total_duration': 2524.9815373420715, 'accumulated_submission_time': 2287.257420063019, 'accumulated_eval_time': 230.2561206817627, 'accumulated_logging_time': 0.16065526008605957} +I0913 01:50:40.739250 140379747440384 logging_writer.py:48] [9503] accumulated_eval_time=230.256, accumulated_logging_time=0.160655, accumulated_submission_time=2287.26, global_step=9503, preemption_count=0, score=2287.26, test/accuracy=0.985692, test/loss=0.0479959, test/mean_average_precision=0.238066, test/num_examples=43793, total_duration=2524.98, train/accuracy=0.989897, train/loss=0.0337018, train/mean_average_precision=0.331815, validation/accuracy=0.986543, validation/loss=0.0451474, validation/mean_average_precision=0.244019, validation/num_examples=43793 +I0913 01:52:40.251860 140379755833088 logging_writer.py:48] [10000] global_step=10000, grad_norm=0.0187978, loss=0.0342521 +I0913 01:52:40.255315 140414510847168 submission.py:307] 10000) loss = 0.034, grad_norm = 0.019 +I0913 01:54:38.892963 140379747440384 logging_writer.py:48] [10500] global_step=10500, grad_norm=0.0207025, loss=0.0346329 +I0913 01:54:38.896209 140414510847168 submission.py:307] 10500) loss = 0.035, grad_norm = 0.021 +I0913 01:56:38.602449 140379755833088 logging_writer.py:48] [11000] global_step=11000, grad_norm=0.0151515, loss=0.0328005 +I0913 01:56:38.605886 140414510847168 submission.py:307] 11000) loss = 0.033, grad_norm = 0.015 +I0913 01:58:13.254155 140414510847168 spec.py:333] Evaluating on the training split. +I0913 01:58:39.788377 140414510847168 spec.py:346] Evaluating on the validation split. +I0913 01:58:41.833229 140414510847168 spec.py:363] Evaluating on the test split. +I0913 01:58:43.919941 140414510847168 submission_runner.py:516] Time since start: 3008.19s, Step: 11397, {'train/accuracy': 0.9905033425243241, 'train/loss': 0.03171524286825818, 'train/mean_average_precision': 0.37856228803121666, 'validation/accuracy': 0.9866564481926615, 'validation/loss': 0.044947521186217074, 'validation/mean_average_precision': 0.2540324694597763, 'validation/num_examples': 43793, 'test/accuracy': 0.9858512060334945, 'test/loss': 0.04768449041747281, 'test/mean_average_precision': 0.25288343198344315, 'test/num_examples': 43793, 'score': 2738.389086484909, 'total_duration': 3008.1850967407227, 'accumulated_submission_time': 2738.389086484909, 'accumulated_eval_time': 260.9218719005585, 'accumulated_logging_time': 0.1932201385498047} +I0913 01:58:43.944404 140379747440384 logging_writer.py:48] [11397] accumulated_eval_time=260.922, accumulated_logging_time=0.19322, accumulated_submission_time=2738.39, global_step=11397, preemption_count=0, score=2738.39, test/accuracy=0.985851, test/loss=0.0476845, test/mean_average_precision=0.252883, test/num_examples=43793, total_duration=3008.19, train/accuracy=0.990503, train/loss=0.0317152, train/mean_average_precision=0.378562, validation/accuracy=0.986656, validation/loss=0.0449475, validation/mean_average_precision=0.254032, validation/num_examples=43793 +I0913 01:59:09.341238 140379755833088 logging_writer.py:48] [11500] global_step=11500, grad_norm=0.0169753, loss=0.0353803 +I0913 01:59:09.344325 140414510847168 submission.py:307] 11500) loss = 0.035, grad_norm = 0.017 +I0913 02:01:08.493326 140379747440384 logging_writer.py:48] [12000] global_step=12000, grad_norm=0.0149312, loss=0.0309349 +I0913 02:01:08.496681 140414510847168 submission.py:307] 12000) loss = 0.031, grad_norm = 0.015 +I0913 02:03:07.720756 140379755833088 logging_writer.py:48] [12500] global_step=12500, grad_norm=0.0166224, loss=0.0304455 +I0913 02:03:07.724627 140414510847168 submission.py:307] 12500) loss = 0.030, grad_norm = 0.017 +I0913 02:05:06.991347 140379747440384 logging_writer.py:48] [13000] global_step=13000, grad_norm=0.0160947, loss=0.0311039 +I0913 02:05:06.995117 140414510847168 submission.py:307] 13000) loss = 0.031, grad_norm = 0.016 +I0913 02:06:16.547213 140414510847168 spec.py:333] Evaluating on the training split. +I0913 02:06:42.801793 140414510847168 spec.py:346] Evaluating on the validation split. +I0913 02:06:44.802654 140414510847168 spec.py:363] Evaluating on the test split. +I0913 02:06:46.831420 140414510847168 submission_runner.py:516] Time since start: 3491.10s, Step: 13292, {'train/accuracy': 0.9909148422903182, 'train/loss': 0.0301297743295072, 'train/mean_average_precision': 0.39912657319498523, 'validation/accuracy': 0.9868740237514573, 'validation/loss': 0.04422345009888322, 'validation/mean_average_precision': 0.27054262197357337, 'validation/num_examples': 43793, 'test/accuracy': 0.9860137680787103, 'test/loss': 0.046997974396899025, 'test/mean_average_precision': 0.2609902563106287, 'test/num_examples': 43793, 'score': 3189.6078889369965, 'total_duration': 3491.096622467041, 'accumulated_submission_time': 3189.6078889369965, 'accumulated_eval_time': 291.20612835884094, 'accumulated_logging_time': 0.22724652290344238} +I0913 02:06:46.855723 140379755833088 logging_writer.py:48] [13292] accumulated_eval_time=291.206, accumulated_logging_time=0.227247, accumulated_submission_time=3189.61, global_step=13292, preemption_count=0, score=3189.61, test/accuracy=0.986014, test/loss=0.046998, test/mean_average_precision=0.26099, test/num_examples=43793, total_duration=3491.1, train/accuracy=0.990915, train/loss=0.0301298, train/mean_average_precision=0.399127, validation/accuracy=0.986874, validation/loss=0.0442235, validation/mean_average_precision=0.270543, validation/num_examples=43793 +I0913 02:07:37.339360 140379747440384 logging_writer.py:48] [13500] global_step=13500, grad_norm=0.0155233, loss=0.0294452 +I0913 02:07:37.342674 140414510847168 submission.py:307] 13500) loss = 0.029, grad_norm = 0.016 +I0913 02:09:34.759184 140379755833088 logging_writer.py:48] [14000] global_step=14000, grad_norm=0.0190781, loss=0.0300727 +I0913 02:09:34.762545 140414510847168 submission.py:307] 14000) loss = 0.030, grad_norm = 0.019 +I0913 02:11:30.710639 140379747440384 logging_writer.py:48] [14500] global_step=14500, grad_norm=0.0171732, loss=0.0287043 +I0913 02:11:30.718641 140414510847168 submission.py:307] 14500) loss = 0.029, grad_norm = 0.017 +I0913 02:13:25.598982 140379755833088 logging_writer.py:48] [15000] global_step=15000, grad_norm=0.016204, loss=0.0277313 +I0913 02:13:25.602430 140414510847168 submission.py:307] 15000) loss = 0.028, grad_norm = 0.016 +I0913 02:14:19.426579 140414510847168 spec.py:333] Evaluating on the training split. +I0913 02:14:44.788327 140414510847168 spec.py:346] Evaluating on the validation split. +I0913 02:14:46.790560 140414510847168 spec.py:363] Evaluating on the test split. +I0913 02:14:48.821494 140414510847168 submission_runner.py:516] Time since start: 3973.09s, Step: 15236, {'train/accuracy': 0.9914435874576981, 'train/loss': 0.028296213395269654, 'train/mean_average_precision': 0.44738064563564217, 'validation/accuracy': 0.9868715882041573, 'validation/loss': 0.04422238771822504, 'validation/mean_average_precision': 0.2697428412578408, 'validation/num_examples': 43793, 'test/accuracy': 0.9860491442750786, 'test/loss': 0.04709520629376225, 'test/mean_average_precision': 0.2617299438584322, 'test/num_examples': 43793, 'score': 3640.7936511039734, 'total_duration': 3973.0866956710815, 'accumulated_submission_time': 3640.7936511039734, 'accumulated_eval_time': 320.60099935531616, 'accumulated_logging_time': 0.26543426513671875} +I0913 02:14:48.844926 140379747440384 logging_writer.py:48] [15236] accumulated_eval_time=320.601, accumulated_logging_time=0.265434, accumulated_submission_time=3640.79, global_step=15236, preemption_count=0, score=3640.79, test/accuracy=0.986049, test/loss=0.0470952, test/mean_average_precision=0.26173, test/num_examples=43793, total_duration=3973.09, train/accuracy=0.991444, train/loss=0.0282962, train/mean_average_precision=0.447381, validation/accuracy=0.986872, validation/loss=0.0442224, validation/mean_average_precision=0.269743, validation/num_examples=43793 +I0913 02:15:49.296016 140379755833088 logging_writer.py:48] [15500] global_step=15500, grad_norm=0.020017, loss=0.0333224 +I0913 02:15:49.299173 140414510847168 submission.py:307] 15500) loss = 0.033, grad_norm = 0.020 +I0913 02:17:43.163309 140379747440384 logging_writer.py:48] [16000] global_step=16000, grad_norm=0.0222029, loss=0.0341091 +I0913 02:17:43.167943 140414510847168 submission.py:307] 16000) loss = 0.034, grad_norm = 0.022 +I0913 02:19:36.602185 140379755833088 logging_writer.py:48] [16500] global_step=16500, grad_norm=0.0178688, loss=0.029432 +I0913 02:19:36.611056 140414510847168 submission.py:307] 16500) loss = 0.029, grad_norm = 0.018 +I0913 02:21:30.825614 140379747440384 logging_writer.py:48] [17000] global_step=17000, grad_norm=0.0178401, loss=0.0285211 +I0913 02:21:30.828922 140414510847168 submission.py:307] 17000) loss = 0.029, grad_norm = 0.018 +I0913 02:22:21.377157 140414510847168 spec.py:333] Evaluating on the training split. +I0913 02:22:46.644495 140414510847168 spec.py:346] Evaluating on the validation split. +I0913 02:22:48.666046 140414510847168 spec.py:363] Evaluating on the test split. +I0913 02:22:50.754382 140414510847168 submission_runner.py:516] Time since start: 4455.02s, Step: 17220, {'train/accuracy': 0.9920402604089243, 'train/loss': 0.026127167900357063, 'train/mean_average_precision': 0.49616984273163683, 'validation/accuracy': 0.9869357242830561, 'validation/loss': 0.04444915683382098, 'validation/mean_average_precision': 0.2739300672967146, 'validation/num_examples': 43793, 'test/accuracy': 0.9861196860952176, 'test/loss': 0.047152139859792344, 'test/mean_average_precision': 0.26818404313760036, 'test/num_examples': 43793, 'score': 4091.952267408371, 'total_duration': 4455.019567489624, 'accumulated_submission_time': 4091.952267408371, 'accumulated_eval_time': 349.97821378707886, 'accumulated_logging_time': 0.29807329177856445} +I0913 02:22:50.778447 140379755833088 logging_writer.py:48] [17220] accumulated_eval_time=349.978, accumulated_logging_time=0.298073, accumulated_submission_time=4091.95, global_step=17220, preemption_count=0, score=4091.95, test/accuracy=0.98612, test/loss=0.0471521, test/mean_average_precision=0.268184, test/num_examples=43793, total_duration=4455.02, train/accuracy=0.99204, train/loss=0.0261272, train/mean_average_precision=0.49617, validation/accuracy=0.986936, validation/loss=0.0444492, validation/mean_average_precision=0.27393, validation/num_examples=43793 +I0913 02:23:54.954312 140379747440384 logging_writer.py:48] [17500] global_step=17500, grad_norm=0.0162622, loss=0.0253394 +I0913 02:23:54.957522 140414510847168 submission.py:307] 17500) loss = 0.025, grad_norm = 0.016 +I0913 02:25:47.787787 140379755833088 logging_writer.py:48] [18000] global_step=18000, grad_norm=0.0196909, loss=0.0258581 +I0913 02:25:47.791042 140414510847168 submission.py:307] 18000) loss = 0.026, grad_norm = 0.020 +I0913 02:27:40.330164 140379747440384 logging_writer.py:48] [18500] global_step=18500, grad_norm=0.0195962, loss=0.0271452 +I0913 02:27:40.333287 140414510847168 submission.py:307] 18500) loss = 0.027, grad_norm = 0.020 +I0913 02:29:33.617615 140379755833088 logging_writer.py:48] [19000] global_step=19000, grad_norm=0.0195315, loss=0.0285482 +I0913 02:29:33.620873 140414510847168 submission.py:307] 19000) loss = 0.029, grad_norm = 0.020 +I0913 02:30:23.242651 140414510847168 spec.py:333] Evaluating on the training split. +I0913 02:30:48.327406 140414510847168 spec.py:346] Evaluating on the validation split. +I0913 02:30:50.311675 140414510847168 spec.py:363] Evaluating on the test split. +I0913 02:30:52.352745 140414510847168 submission_runner.py:516] Time since start: 4936.62s, Step: 19219, {'train/accuracy': 0.9922587325620501, 'train/loss': 0.02521319431163083, 'train/mean_average_precision': 0.5292838896308407, 'validation/accuracy': 0.9870345669109791, 'validation/loss': 0.044852398476240424, 'validation/mean_average_precision': 0.27909230434120236, 'validation/num_examples': 43793, 'test/accuracy': 0.9861563257271704, 'test/loss': 0.0476955290278537, 'test/mean_average_precision': 0.2709051437065942, 'test/num_examples': 43793, 'score': 4543.044823646545, 'total_duration': 4936.61790394783, 'accumulated_submission_time': 4543.044823646545, 'accumulated_eval_time': 379.088228225708, 'accumulated_logging_time': 0.3312056064605713} +I0913 02:30:52.376145 140379747440384 logging_writer.py:48] [19219] accumulated_eval_time=379.088, accumulated_logging_time=0.331206, accumulated_submission_time=4543.04, global_step=19219, preemption_count=0, score=4543.04, test/accuracy=0.986156, test/loss=0.0476955, test/mean_average_precision=0.270905, test/num_examples=43793, total_duration=4936.62, train/accuracy=0.992259, train/loss=0.0252132, train/mean_average_precision=0.529284, validation/accuracy=0.987035, validation/loss=0.0448524, validation/mean_average_precision=0.279092, validation/num_examples=43793 +I0913 02:31:55.974288 140379755833088 logging_writer.py:48] [19500] global_step=19500, grad_norm=0.0189949, loss=0.0294257 +I0913 02:31:55.977680 140414510847168 submission.py:307] 19500) loss = 0.029, grad_norm = 0.019 +I0913 02:33:48.200229 140379747440384 logging_writer.py:48] [20000] global_step=20000, grad_norm=0.0174147, loss=0.0261211 +I0913 02:33:48.203579 140414510847168 submission.py:307] 20000) loss = 0.026, grad_norm = 0.017 +I0913 02:35:40.237685 140379755833088 logging_writer.py:48] [20500] global_step=20500, grad_norm=0.0189442, loss=0.0280141 +I0913 02:35:40.240869 140414510847168 submission.py:307] 20500) loss = 0.028, grad_norm = 0.019 +I0913 02:37:32.088920 140379747440384 logging_writer.py:48] [21000] global_step=21000, grad_norm=0.0227868, loss=0.028208 +I0913 02:37:32.092119 140414510847168 submission.py:307] 21000) loss = 0.028, grad_norm = 0.023 +I0913 02:38:24.977591 140414510847168 spec.py:333] Evaluating on the training split. +I0913 02:38:49.831026 140414510847168 spec.py:346] Evaluating on the validation split. +I0913 02:38:51.812797 140414510847168 spec.py:363] Evaluating on the test split. +I0913 02:38:53.812364 140414510847168 submission_runner.py:516] Time since start: 5418.08s, Step: 21236, {'train/accuracy': 0.9925139913291285, 'train/loss': 0.024142121883480805, 'train/mean_average_precision': 0.5400073620419426, 'validation/accuracy': 0.9870051373811047, 'validation/loss': 0.04515812625938092, 'validation/mean_average_precision': 0.28139374424142705, 'validation/num_examples': 43793, 'test/accuracy': 0.9861407433549605, 'test/loss': 0.04821267558596037, 'test/mean_average_precision': 0.27437666614556744, 'test/num_examples': 43793, 'score': 4994.298848628998, 'total_duration': 5418.0775537490845, 'accumulated_submission_time': 4994.298848628998, 'accumulated_eval_time': 407.92298674583435, 'accumulated_logging_time': 0.36398983001708984} +I0913 02:38:53.836669 140379755833088 logging_writer.py:48] [21236] accumulated_eval_time=407.923, accumulated_logging_time=0.36399, accumulated_submission_time=4994.3, global_step=21236, preemption_count=0, score=4994.3, test/accuracy=0.986141, test/loss=0.0482127, test/mean_average_precision=0.274377, test/num_examples=43793, total_duration=5418.08, train/accuracy=0.992514, train/loss=0.0241421, train/mean_average_precision=0.540007, validation/accuracy=0.987005, validation/loss=0.0451581, validation/mean_average_precision=0.281394, validation/num_examples=43793 +I0913 02:38:54.276631 140379747440384 logging_writer.py:48] [21236] global_step=21236, preemption_count=0, score=4994.3 +I0913 02:38:54.399078 140414510847168 submission_runner.py:857] Final ogbg score: 4994.298848628998 diff --git a/logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..f3d2a900 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,13 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/accuracy,test/loss,test/mean_average_precision,test/num_examples,total_duration,train/accuracy,train/loss,train/mean_average_precision,validation/accuracy,validation/loss,validation/mean_average_precision,validation/num_examples +78.37495589256287,0.0,31.56527137756348,1,0,31.56527137756348,0.5467212161999395,0.7414811802846774,0.0282347550766273,43793,110.37323498725893,0.5586510674202247,0.7345021011061005,0.0234666014732334,0.5490050383355145,0.7403665985795889,0.0263411271506255,43793 +108.7675120830536,0.0296380519866943,482.64182305336,1909,0,482.64182305336,0.9837648527381596,0.0583073679772985,0.1074296865913766,43793,593.2517759799957,0.987392573965418,0.0462614535726064,0.1083306340867504,0.9847544887136738,0.0551702901690756,0.1086062017457213,43793 +138.99151515960693,0.0630345344543457,933.7262935638428,3823,0,933.7262935638428,0.984448371389417,0.0535538692609491,0.1640744205277744,43793,1075.9749946594238,0.9884848246424988,0.0404075672711894,0.19120728049328,0.985415130918786,0.0507522517649599,0.1628767457703015,43793 +169.49265098571777,0.0961549282073974,1384.9259967803955,5720,0,1384.9259967803955,0.9848665685679128,0.0509523698471832,0.2026070270414677,43793,1559.084408044815,0.9886374956736488,0.0385616442536409,0.23909901151243,0.9857841163347286,0.0481692713999363,0.2052480629121319,43793 +199.7694013118744,0.1288614273071289,1836.0931000709527,7610,0,1836.0931000709527,0.9853134036196588,0.0486191333421493,0.2286057515900184,43793,2041.9273238182068,0.9895910571669182,0.0351091206909363,0.2911635532020825,0.9861514780524714,0.0460630208773084,0.22996665595578,43793 +230.2561206817627,0.1606552600860595,2287.257420063019,9503,0,2287.257420063019,0.9856915920046428,0.0479959305792683,0.2380658957450744,43793,2524.9815373420715,0.9898969062830736,0.0337017683760116,0.3318149727723656,0.9865431952432138,0.0451474168280893,0.2440190009526703,43793 +260.9218719005585,0.1932201385498047,2738.389086484909,11397,0,2738.389086484909,0.9858512060334944,0.0476844904174728,0.2528834319834431,43793,3008.1850967407227,0.990503342524324,0.0317152428682581,0.3785622880312166,0.9866564481926616,0.044947521186217,0.2540324694597763,43793 +291.20612835884094,0.2272465229034423,3189.6078889369965,13292,0,3189.6078889369965,0.9860137680787104,0.046997974396899,0.2609902563106287,43793,3491.096622467041,0.9909148422903182,0.0301297743295072,0.3991265731949852,0.9868740237514572,0.0442234500988832,0.2705426219735733,43793 +320.60099935531616,0.2654342651367187,3640.793651103973,15236,0,3640.793651103973,0.9860491442750786,0.0470952062937622,0.2617299438584322,43793,3973.0866956710815,0.991443587457698,0.0282962133952696,0.4473806456356421,0.9868715882041572,0.044222387718225,0.2697428412578408,43793 +349.97821378707886,0.2980732917785644,4091.952267408371,17220,0,4091.952267408371,0.9861196860952176,0.0471521398597923,0.2681840431376003,43793,4455.019567489624,0.9920402604089243,0.026127167900357,0.4961698427316368,0.986935724283056,0.0444491568338209,0.2739300672967146,43793 +379.088228225708,0.3312056064605713,4543.044823646545,19219,0,4543.044823646545,0.9861563257271704,0.0476955290278537,0.2709051437065942,43793,4936.61790394783,0.99225873256205,0.0252131943116308,0.5292838896308407,0.9870345669109792,0.0448523984762404,0.2790923043412023,43793 +407.92298674583435,0.36398983001708984,4994.298848628998,21236,0,4994.298848628998,0.9861407433549605,0.04821267558596037,0.27437666614556744,43793,5418.0775537490845,0.9925139913291285,0.024142121883480805,0.5400073620419426,0.9870051373811047,0.04515812625938092,0.28139374424142705,43793 diff --git a/logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/trial_1/events.out.tfevents.1789261715.f68f7f208e51.38.0.v2 b/logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/trial_1/events.out.tfevents.1789261715.f68f7f208e51.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..4fe000a62cfc35569ab8a890471560211346f78d GIT binary patch literal 45446 zcmbW=cR-C_0|)TP=w3-e8b&fgB%??}$;is45@|^^s7NV7k%S^-C1hkv8idTUGm4B- zRuan0eqZ;=IrsaF+)`L>?&s?p=$$0x)Ha_{-kwoBW^9O+^35VsSH+fZ`sl2_iLx zJIw!iM{x&n12+$ECr1wlUq9Dbec4{h5<)j?T_=d>owv{^p#MZ-kZ%gW*?tRpm_S+1y@aJeK6>h}(?HXLnzBZ?C^sreKZ_`x5`YbQl}27C=Fsvc~6`5uDx?|N>u zsEU?LJhxvoJ&;8A688`{mx}h9>)CtOjDb~G^7_T} zBch#AB%+VlP^>A5sD8fI4u6UB6=IuOY+Mzp(e{A7D0c%1)f0CSHgZjktAEU5Q7jkhxT+aE}zSH%pk}h7I_)U}&cKq+o$~hZs>lov8 zqz5kS2@lJyC^DM=bf{#X4p)>L8o_)N#gYh_=R>c>@uQ$PJpUr2sLeg$Ofp#~+@iMb zCqSVeN=EZ5zOWU|*Jf-4g<~X%5LU>U!>zHQbv;{9l<4O&3VE)g-!hiMbIts>pxDrd zjOJHx*5cUEb^}21!kk11E6h>RymaC{P(;*y&M4||6#;Q9#kUNz6`;7^kc^g5QE%6% z5Kt60BN4)i5LEmQ-LC|Shf6;)in?6Ik~AZHE?jS#s)E9EH5tvX;M`G87H1!VB693$6~h zGO@q}6xJP@F^c+Jg?vsQT#K&VWAs6hqDMyaD>!R$V(j5OP!t-H2w}w&RD2ru{yr!! zl(b_M3S7nTHY~*xz00|v`1OH|=2v`W?@@b($J7T!;13cZtdPUQe|_&C7eS%By%(bp zaTON~3~?<^@3iv+#ok0RnqR?L3rp{eMWDzyN+N_6%BW}@eGyN+?~?~G3Ncr)MTw=j zJ}R*k6!%@pXnqA}rw#VGe;yQDyhwzwLJJk0of@8kwfJE;j!`t=Do*Y)z_sYz@WTU8 zyzfUw%cwAqTeTV#C#^_?utFad%bvzh1;tne7e=ATRje=UiHhvEVVa!#l zh3|*)pjfI&B7_y@sIZ;eeJChG%6%C{L#{%lPY+bMcigQGityJ?1lLfbouQ8eZ% zy5#Gk;!gC3&7k-)myG6DaPBChgtwnTu{(rB2rI%-F-xIB6%;!fo?{d$T!qqNmSWXp zlc}Is*o=(kS8(nq`9bk3Kry=&i4ayqq2hFlI1Ny&NVvr)ns60|zjea32>d$xGbrQ^ zlF|H%PwWO_Q<-WG3hUz}LRb-tiVU48tw6EQsfbZDl?E#8G4rDaH;v@UHh|X#20g7-p5+STeLxudJW7j|- z>Z8Rd)VPY2Au=eK#}fEM)NB^u%GUN z(rx$&XRbeq5LP@v#j%BP+dy&rrx&AW$yH>gG)2Xg)yjsTm{rgF?`VF-d$!{J`LLs~ z7U4=HLRe9P3ftA@*lF{RFJP{t6<1-cqJoNN>nAya;^!$cnqR?L3%M4j4M1Uki9`r1 z%281mlO_lqtlipt4o85NlWJ{N#u{#Ozqtf+&EI}tB$fkHj&G^5bwDv}3@ zaV>0`Mg)N3{Y^5OU%|Pf9^RUs42stGNrbRM0Tt0-m(=rcUyh{h4Xt7A*@hFg>lZY;h@NB^o~)q=PDXblt;z;ZG&Eb;7xySFu@54i&xQmf&aRI*rL_e#IO1T3jz$Hyac_>Lfx~ z(E=5{e7)0%E5EKJOTQdq> zuA+G1XH+DN-XRZ)S@vW!zv2a3(W5~`0IbDT7ZM??=z@wa+RyOQ=DB^QFp4f*MF-ys zRQzf+(*hKa&BrR6bXt3DP%OiqL{rFKW4?J zgJS$?5+STGLB*<~?^>WRT)UM~=yMgXKfXkTw*QJmP~>bTqxls@Y(-C;@5ljH3bATl zZCH!!Nv9Y^53b^e@iSbDf_6Um^Th-gGMZme$X<&{g+}?Hi18p1!U`)?bnevye}BwaCyWqxlt_JL>A@vlBp()R{yGD~6*YL%k{ftmj(vh*9+7 zDxR4=LB+n4r>}!zQz;qEuP9)zMXE>8R#3QoAQ8ffQK)DYHhO^e9 z@iD@ox4HMq$WRbba**FCo6k zh?k&PA4o>?E3!MWYVo^OyXS&JX9Ye}c)YB{h@Lj%)j=`CnnVaI>`^h_J)u6VMY~*0M$w0>7@m0_FX7?Ur^`Wc z=`k72uUOTORVYW)3d;b+m**rxSTPk9HhGteK%pI>%P5SviaCu~YoYRS<0DXPUPDIn zE7aDr3gz1U+EJi5znMe`D_l@<(kA{RD4LBiW)yw73YWh3a4n|3%{&T<>2_o^zrxb~ z9Y(lTzS{(fI42SzteB39;2C{WL2;`0Fh*g_RUCevhnLW|-D&K!ZOqANenq+&t56!6 znXUoFKr0do5!#g4V#nE{0gOBWf(Ce^v6z6 z+-glCgcY+;u`1109TY!L&R`U#Tt(x>xA77l9MP-=iVYXZXnw`fey=g2EN|3pP_(~6 zB7_xwsIV*8x(^f^LjoCv8CNlNQVw3i?;&0AlbnOg$!LB>^ARjXLxrLEn_~4fBtlp* z4;6}Ho9>`kGAe>m^y4bz%~^_u-9Ai*weXlkM)ND4h+g4D_`W`s3TrXNnM4RHf>5Ed z>~j_`8>MA_NuRUMsc- z#ijxK7zNLF^MH;a{_CwjmM1ip4BN_gMT4#Hz;$ zM$wc1$_cbP?`4t@oJjIB+lTY1+wJ^0L5yFa4RQ!G#)(h66@srDp z!jh}7Kh9Fz_^EIc)?#M~8O^VFtW$uBHF_rRLE%tNB7_x7QBl03moF$LpUY!hHw@6H&}}0;&-!QEu3$Y(fo={ z`YajEuV}H2^&?6< zgZ0Ir$hb@*gca*iF}8skw!z#I8%8mltN7gaEGojjfBOMPWTgR?%d^!VkB2F*Y^}&!s$(V^IB%FnV|R{O(KL9@u*nUCf*0` zs8F{=MlqVJD9t*Fil(;);BW13c$3lmiua=H7-9dZegY_p{YiwdVh1Y59?!oHiV@c7 zjA9H|v1v7HEt;m?J^*X+Wda$^uUKltD)&ln+)P1{J%vOFD-uxgBJmpj{vuK3GNTyF zRUB13ffLzFZa@BfaZ!_u=2z_5c@-mS$48$8MfY|jLRhgI6;sQ?X2V((w8>`_B&`=l2?2F3e+BtlrR4;AXy)$upQ&n>?)iV0jr{?;^Hi*F}v zmVsjaa59=-5m~@0lsc)V6G0I;jzkD64xqwe{r*j`7Tbr2>PddSn8;Ne@kv3&(YKZO z`|e9)$Y_4WV`J72S{WNsDnU_fM+YiVaru)Kgd!nGxD>BJ1S)`8O^VV zdB-aEUe8rKgTiYRi4ayKp(0&DD*@J`leQ6~u;VJW4Nk^8>WJA&Ng{+5$*34n_sB_5L>LWW6qC4$ZVD{L&>3yZLD9WG8O^U4mcUvI1BH3o zptw4eLuGC z5+STeLq*ovCOV+_Rz96kOy(-udnBQvV%&mLplJAwjOJI=+09ZkT%n48d$6{)``-%? zR-8aZ{*e=RK#}o!9uqQ!t0)+E6cugSjJ*tsU7yHke#L%6mf~LWflycrt6wBSSdoDW z{{xRcfg<2#IHPdjDmt$@f{Nv*N2~-zi*hoWU(vyX_2Xi|YDXPVs8o^&VZ~`weAO1= z->R*uw~2T{?_?ujob%pZ}_{EBx=Ssy^Sg|_boii^)kgs|cqDyBC%+W-_!JJJ}1 z6IZd#hNZYLCTc$@lJ}F*{EGA>){pMF-8*E1;{7oaA*{H7iX;#DP*7a=&t??PTt$0} zL%0?;DupqicoafL^DE>sSc+)P#BreTT23N_6_-%aPjMuEbHUE_KBI8qD#pKHtwmsA z6Z~5>*I8sVzam|S^#Mek?n!+>F>4-)5LRTPBFlA2J$T1n?z)Utl1}@P=Ve|h87oZT zRbX)z=_}zym059gIKM;e^%dV3cp*11-LUG_ZX+i~z7x6bp&7!EZ2Q8XhoNx@8P#P%eSmyuO*4ccfdc@t{dTYsK;qu+ zu$4d_@w_g_EV(yA-WaVd1jx}ZbT~g`;j@znIr6lTAwZ_e?If2V2njifkb>|luJCqG z<4fiN638c|YTt%>F$lj|qT^et05|&392z9>I@H}krZ_Jje|er9Ty>Y7IRAaKsMmcO zBJ=xD{T@BW*xqB$wK1!TmAwDfypYEJ>xHO=YAorw^XL8HeTj2k(6s8_h-&uD5wGiP zgH6%HtFG(Q-5v2Fk2f@_X0Jqj>rsH&t~c$@!pkPFex%v|>7A(8^XvD5S9M;WxGzT48B>8KTOq19U3I-t5G?ZcYa>RBJ(O<$L1f`iKgk%)PKJl^-Fj@9&x$L zmgiembO|Jl=WQ)Z9DCtK)EV4986eNB=x~0>x_ujQMa&vM zNCC*+(KJI45@L*yNd;HV0Yv-U?0XW(0iM@uxP9!45bXy3c>tMxl@8~JeD;2XkQ9Yd zELNs>X@(#q(+(jK%C3W9+uPLOU4;blu4Q$7WMac$JY`N=cE=Aew>PCh0)1rq;`TT% z157WdK_4m9q#-i*k$YDy@uwrjuRe{TkK}68*njmA-2tpV68B)reCQ)hI@7f3`bcci zNaV`R{4@djNK_A+RI@(P>2e1&f9>!+8v2M_Uz+`&`pBfXIr`8?-kQ@y;XZP+4r~72 zcfQ*aZvKu|G)HptbNa}LNt1Bq0(Czxf!R>s#YE}NY*~Wxx7QA zKW;%ES?Wtu|J_IK{uE>VdVar6W9TC*18JU2eZ*XC5#A&EgI%A(PTyrQjTGD<%e`=e zZ0g;&68cEOa+*@JKGGoadLuYAdbHT~L$cS#^SpaTwW0@l!tZP1>F3;@4(C5KoDB2u z&^X~aDI4ZwV^^9XI45#}I45gv*{=e~_lZFo3;+C!9x*&G3%SyBFhV9~L|uaoppg?D z&JQuzorRFKKQ}c6h=vEv5QM}>B4nnVTql4G+2Ptp0tx4NyF<}!){2ZYZuAl$p8M%= zeu(3#1cWqssnro6E0bx4AjEtNLT1dar~pXH=MUp0kRLoR707I+i7V2~xl9KjOMlbh z{E)0eKM?Xf;Jr0KG88_NOAv%S*^dz8$;URswl`KjWrhTDze{y}#I`xBk6hBtFNM!X zEEH*wKp(mFz9r5}>$B7Jp^ua_p&>H&5r;fKJZCCDr0;+}BG-b({;QAJ+Oy8nsI6JM zp^q$TOVg_BBke{EMDB+sjR!#=+1i;V)vS+1b{T`_iWQ3jp^upKpxOVak90Qk*$I6l z(1<1q_mSROtobuAKCvBc{+;GDM{@IX`iS?jQ8;tI>mKV0ePsJ!njvc+={;{G&RKA{ z%@OD$CDt^xhJECcC+8Jz11edNqyno_eqQuy#neK<5$KS`V~*=x&r-nen6yEb~hM~#z* z=f>j|9nODfcv`Yfzg`{ZWx$+xSI`W>ISDz5a}wk#8LOlKEhn2SgNcYJ!LlB~M6(L_{C1(R9<-T1ibNcaor~QmdBU};n zmu`ara-)b2=Z6?{OG1e0qJY-`=~_xN1ROx~2$@=n5bvgM%3<5{d^z)}1aiW-x;}DmJgblR zzIucIu+6%18YIw1*4$2UwrLw#u$u7y8J&hBT>WeZ(StbHWD z`3#&h-O{|F&_{H8(9|0Ck%b$^bc8$+edNo9!PrOaEn*%*A90;P^JMBH<{#y8 zHRJV$7{CVUF@;77Zjd=|aD$w7rP3YxNQx^>saYSXFvxoZhenUO`QIdaZ6?n*^b`FX z;GvN7V#b0YT*=Ok!r>J)%+!j;BKpeX z#z`Psc)qk~E>c3s&DZtM0c5p09nKHQs`NvM_hUP(x{I`Fh9G2CQ-nAaq^*H%&uV5j zdkJLQ(CYd~u{*1e7*Ds#gwIEI`q3bPK2jhSm(bXM^%1pItn;*eW$AwCBjph^t-3z4u&fVq$FG#jgg){mnkLn(kFrYafZs^2IrufAf<%^bw0xnp(p?GAJ@w9{R|PGc=3aM_PSU!ae`j>dgu8 zV7gwSjRLk{nIx{AfY^8!R-66Awq7gc)SQ8`orjOeu!0UBA$M4l2Ww+vU)7d5QHeVLx|H) z@7e%4vVZq+38a$en}3&W>mg*Sd-LZ2$xf!j`5~Rm<{_k4X>ePBJUmS^1R+m!5fbsT zO&i!PwnYB8D}jXBR(FhVk7ga?oqm+!w|oYzr$GY8`2Cl)a9&)kBQ@X{U$>Qp$b5{y zzRmjU{`Wg<#=lhDR$2x|N%)0&#j`4#1G_AU0{Fw&p7;o&;s1F?D7m{dF z&5rRE%8PLmGi~0cBOK#fPtffDbc|n}IzttX@!scXqVO?(W-4p`G{b-IhnwH!GR@&X z#&bql;*B!6lX^CsO+#~NhOEbUdB7r^v)Y%M7{D4pWyfq_s*1{JkgUfz=kjK6u)YGv`1ubs_20+1 z+}y=@j1PA_xF3%3d*5iDOviZ0!-2S(i+gO3ferFq?Vn_86Wk!1_P`A?qp3zV9OK=^ zG^J+8_@!-UTfw2>_GbJ;$zEH=^R?I)rOJ5E-!r;t1kcT`N;;hX(74@b1s)nDaSKE+ zCwps6AeSIGCr5hXoOFqKXbh0jL#@9`AWwL{$Jlk|PrL|GdDt+3>`tS@`5~=$uzrZ0 zHe&rjfasj18G?`yV}z_Zu{9PTE&9JzUG(Skk-I!!JS&{e`fiNc&ISDdGGzoE&JWqH z8;+9^b<@TlAa!hLh9G2=B|_F;YPA_4J)+O_l0dRux!-qMas3xwgi~UP3qW>nrNj9l za`{UzBgo`?bAW8wLo);+TEh@BH-B#?Y=SS2zAJ^pSoBG(_e;GTM&y9*~d}^?J}p0!(P^zxqhlPS$yPc-!iF&`0tu zX`pUe?IWS_VK`@^0IeI) zM~r9D)Ef4YNvdsTLLW(+N3*zn^DtPxR;3+3U#Oj=0CSJOu#yR-h#yN||kPc9Oj|!<~C*X!owb&9UN9As!l) zkLhs!L&IwP0?bJ3bh|bj8gpLI48b`mAA@sZsXnR>Kz1Y@@s~i3cyb|iC$V<sKfMMCb9r7zrd{HWxCX{cF64*tEm} z07-s9hx0?y4n<-{s~aoU0p$KCnjr{@azcprN7E|+x$Habs00!_hYPvf421Mi-0Bb8o|2y7B?%R{ zx5-^Gq(K6GWbXBsXj-S`)DHT{MN=9gb02Ba=sE68^Jnhf0)1rP02=$RJ`%_7BV!w_ zegl2PcNk5pu8%C&X7!PV6+IQ8k2D@blWNvSGBjCzB%x$bYv?0tb~O7x^^s0b?|VQW zIXRUk3ipvzPgWoCZ~I^d-29<#G)HptbNa~37py+=YI3$d9J1P8G(*-tGOyNByicyh zHwuD2QfCfLtzjQ|Z`E%%^pVd&G>h9ujzv0SAIX|imIx1K?sGXr$l6y5gamUTUyif> ze#B_75&l5pa)=J+hukkdj*wyBp7#VubSljdgvbRVPg(7_1}QtY8r<$J$NcAI$rW4C2^FajK}$&pTE2UZ=?j)F zACo&vEG3~lmgc)&;boNjO??O-C_MckHJ;z%)ShKI75OR=ES>5erxzh;$q7Zv2KUH| zaKy&FK58bhTv%GGx>{oQnN>@A1X*{2`^4g-RFXh1Nl;*Yn``^Jd3~*_mgv_StF5B( z-=$JyE+$_^zi^AX=vnJqzpC8Pu5%qln`miWci5{^MN)CqRg*Ew zhPc^W^tiToMAemX3>u(TpBau{A6xBPx39cdrWDx@5^K(UHDVVcdpda%LeGi>O5dV&Ng(|Nnn` znDvl)`#E|zxXkf!^mF(2>MvHPr6Q+Yr(>TLm{cUcX(de8ufbADvUU{C1rF5ocxzE5 zluJCfU$n$pD)!$c#qj4LEGgsV&NmrQ6=-Rzw@LJ2oK&DpEk${2EUxL5uBFXj#~f=f zl_|JmhA+h(GcvSK3RKr`PEs*7>nVp@Z83(E&NpfoJ$RY3u zE$A&Zp8q82-G}uDqQR?=9D{jrF_#JvoR_q4oR{3qN%z3gE&Tlqi6tkJ$KrSQ4qDFd zKZXC?-QFmv@%)z4*4r^8a(T=pu$+mN3K6u#N20~pIVu4xvrU>UV;;CD9?L$50=$e} z(~Dcdy3`*iHJ;xxDm)%T{O9gc0!#RCsSrU+cobST*==kC7TpB5of1o043Fi*V^&)c z4|}{AENKU%#`9ak#J=m`|B*g zcY9pO-ig0sayO7l5@;(gSKPoQ(i(Zg8rq7bsZ@&0ZKcD)`gnDsfHjSvt*8x<%KcYc z8GeX$(%KDroepiKVyIMHb#0|XD(kg#Zj^Uzkprb-QS_oP5WEB_vmBJilea{u4MMfe{ZIfaUmOsSrVn zaspb?PkJ1Ijqud&Io%|dkGraEE7O{>+Dh=)N$23Rm0^dak_6hyv5Oaxb$8JACD2y1 zQlwI3ZYyP`Vr(mCb8kI?wo;KHmHV%@GUWiPt-Mr;F@m6{)D2wUv7+QP@^)MAu4!wle6JRQP{tE8VghuY$G`a$hP^xUHo5vgUBZjNN^Y4{Yz^B=_kc;o z&{pn$lnUdv6&DrOYlfZ$^*;p?LJ?{wlYP-ZZ?Y|`zAhW5DNu>^$YUfME M)XY?`=xE{p0OxJ6`v3p{ literal 0 HcmV?d00001 diff --git a/logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/trial_1/flags_0.json b/logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/trial_1/flags_0.json new file mode 100644 index 00000000..af6cf250 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/trial_1/flags_0.json @@ -0,0 +1,63 @@ +{ + "logtostderr": false, + "alsologtostderr": false, + "log_dir": "", + "v": 0, + "verbosity": 0, + "logger_levels": {}, + "stderrthreshold": "fatal", + "showprefixforinfo": true, + "run_with_pdb": false, + "pdb_post_mortem": false, + "pdb": false, + "run_with_profiling": false, + "profile_file": null, + "use_cprofile_for_profiling": true, + "only_check_args": false, + "op_conversion_fallback_to_while_loop": true, + "delta_threshold": 0.5, + "tt_check_filter": false, + "tt_single_core_summaries": false, + "runtime_oom_exit": true, + "hbm_oom_exit": true, + "test_srcdir": "", + "test_tmpdir": "/tmp/absl_testing", + "test_random_seed": 301, + "test_randomize_ordering_seed": "", + "xml_output_file": "", + "experimental_orbax_use_distributed_process_id": false, + "experimental_orbax_use_distributed_barrier": false, + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "workload": "ogbg", + "tuning_ruleset": "self", + "tuning_search_space": null, + "num_tuning_trials": 1, + "data_dir": "/data/ogbg", + "imagenet_v2_data_dir": null, + "librispeech_tokenizer_vocab_path": "", + "framework": "pytorch", + "torch_compile": true, + "experiment_dir": "/experiment_runs", + "experiment_name": "submissions_a100/ademamix_golden/study_1", + "save_checkpoints": false, + "save_intermediate_checkpoints": true, + "resume_last_run": null, + "append_timestamp": false, + "use_wandb": false, + "profile": false, + "max_global_steps": null, + "overwrite": true, + "hparam_start_index": null, + "hparam_end_index": null, + "rng_seed": -1419473739, + "set_pytorch_max_split_size": false, + "pytorch_eval_num_workers": 0, + "capture_jax_trace": false, + "skip_evals": false, + "?": false, + "help": false, + "helpshort": false, + "helpfull": false, + "helpxml": false, + "tfds_debug_list_dir": false +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/trial_1/measurements.csv b/logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/trial_1/measurements.csv new file mode 100644 index 00000000..be87e62a --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/trial_1/measurements.csv @@ -0,0 +1,157 @@ +global_step,grad_norm,loss,train/accuracy,train/loss,train/mean_average_precision,validation/accuracy,validation/loss,validation/mean_average_precision,validation/num_examples,test/accuracy,test/loss,test/mean_average_precision,test/num_examples,score,total_duration,accumulated_submission_time,accumulated_eval_time,accumulated_logging_time,preemption_count +0,0.4999997019767761,0.7284789681434631,,,,,,,,,,,,,,,,, +1,,,0.5586510674202247,0.7345021011061005,0.0234666014732334,0.5490050383355145,0.7403665985795889,0.0263411271506255,43793.0,0.5467212161999395,0.7414811802846774,0.0282347550766273,43793.0,31.56527137756348,110.37323498725893,31.56527137756348,78.37495589256287,0.0,0.0 +1,0.4999997615814209,0.730535089969635,,,,,,,,,,,,,,,,, +2,0.4999997317790985,0.7294957637786865,,,,,,,,,,,,,,,,, +3,0.4999997615814209,0.7286261916160583,,,,,,,,,,,,,,,,, +4,0.4999997615814209,0.7289806008338928,,,,,,,,,,,,,,,,, +5,0.4999997317790985,0.7254716753959656,,,,,,,,,,,,,,,,, +6,0.4999997019767761,0.7258564829826355,,,,,,,,,,,,,,,,, +7,0.4999997317790985,0.7219240069389343,,,,,,,,,,,,,,,,, +8,0.4999997913837433,0.7192580699920654,,,,,,,,,,,,,,,,, +9,0.4999997317790985,0.7117578983306885,,,,,,,,,,,,,,,,, +10,0.4999997615814209,0.71116042137146,,,,,,,,,,,,,,,,, +11,0.4999997615814209,0.7095600962638855,,,,,,,,,,,,,,,,, +12,0.4999997317790985,0.7026304006576538,,,,,,,,,,,,,,,,, +13,0.4999997019767761,0.696168065071106,,,,,,,,,,,,,,,,, +14,0.4999997317790985,0.6907883286476135,,,,,,,,,,,,,,,,, +15,0.4999997019767761,0.6849061846733093,,,,,,,,,,,,,,,,, +16,0.4999997019767761,0.6787175536155701,,,,,,,,,,,,,,,,, +17,0.4999997317790985,0.6712825298309326,,,,,,,,,,,,,,,,, +18,0.4999997019767761,0.6699532270431519,,,,,,,,,,,,,,,,, +19,0.4999996721744537,0.6639813780784607,,,,,,,,,,,,,,,,, +20,0.4999996721744537,0.6520723700523376,,,,,,,,,,,,,,,,, +21,0.4999996721744537,0.64585942029953,,,,,,,,,,,,,,,,, +22,0.4999996721744537,0.63761305809021,,,,,,,,,,,,,,,,, +23,0.4999996423721313,0.6298992037773132,,,,,,,,,,,,,,,,, +24,0.4999997019767761,0.6271013021469116,,,,,,,,,,,,,,,,, +25,0.4999996721744537,0.615108847618103,,,,,,,,,,,,,,,,, +26,0.4999996721744537,0.604968249797821,,,,,,,,,,,,,,,,, +27,0.4999996721744537,0.5984118580818176,,,,,,,,,,,,,,,,, +28,0.4999996125698089,0.5908957719802856,,,,,,,,,,,,,,,,, +29,0.4999995827674866,0.5809135437011719,,,,,,,,,,,,,,,,, +30,0.4999996125698089,0.5705918073654175,,,,,,,,,,,,,,,,, +31,0.4999995827674866,0.5646641850471497,,,,,,,,,,,,,,,,, +32,0.4999995529651642,0.557313084602356,,,,,,,,,,,,,,,,, +33,0.4999995529651642,0.547977089881897,,,,,,,,,,,,,,,,, +34,0.4999995827674866,0.5401413440704346,,,,,,,,,,,,,,,,, +35,0.4999994933605194,0.5318260192871094,,,,,,,,,,,,,,,,, +36,0.499999463558197,0.5238636136054993,,,,,,,,,,,,,,,,, +37,0.4999994039535522,0.5130470395088196,,,,,,,,,,,,,,,,, +38,0.4999994039535522,0.5094386339187622,,,,,,,,,,,,,,,,, +39,0.4999994337558746,0.5003754496574402,,,,,,,,,,,,,,,,, +40,0.4999993443489074,0.49546018242836,,,,,,,,,,,,,,,,, +41,0.4999993145465851,0.4903895258903503,,,,,,,,,,,,,,,,, +42,0.4999992847442627,0.483528196811676,,,,,,,,,,,,,,,,, +43,0.4999992251396179,0.4814444184303283,,,,,,,,,,,,,,,,, +44,0.4999992251396179,0.4715816378593445,,,,,,,,,,,,,,,,, +45,0.4999991655349731,0.4688979983329773,,,,,,,,,,,,,,,,, +46,0.4999991655349731,0.4645242691040039,,,,,,,,,,,,,,,,, +47,0.4999991059303283,0.4572701752185821,,,,,,,,,,,,,,,,, +48,0.499999076128006,0.4530404508113861,,,,,,,,,,,,,,,,, +49,0.4999991059303283,0.4520061612129211,,,,,,,,,,,,,,,,, +50,0.4999990165233612,0.4457701444625854,,,,,,,,,,,,,,,,, +51,0.499999076128006,0.440786212682724,,,,,,,,,,,,,,,,, +52,0.4999989867210388,0.4379798173904419,,,,,,,,,,,,,,,,, +53,0.4976733028888702,0.4404948949813843,,,,,,,,,,,,,,,,, +54,0.4944593906402588,0.4314664900302887,,,,,,,,,,,,,,,,, +55,0.4861406087875366,0.4271439611911773,,,,,,,,,,,,,,,,, +56,0.4809536337852478,0.4233421087265014,,,,,,,,,,,,,,,,, +57,0.4718482494354248,0.4258991777896881,,,,,,,,,,,,,,,,, +58,0.467500776052475,0.4203673601150512,,,,,,,,,,,,,,,,, +59,0.4633691608905792,0.4197210967540741,,,,,,,,,,,,,,,,, +60,0.4613346457481384,0.4138860702514648,,,,,,,,,,,,,,,,, +61,0.4594703018665313,0.4111485779285431,,,,,,,,,,,,,,,,, +62,0.4595725238323211,0.4069471359252929,,,,,,,,,,,,,,,,, +63,0.4551534652709961,0.4067383706569671,,,,,,,,,,,,,,,,, +64,0.4496554732322693,0.4037118554115295,,,,,,,,,,,,,,,,, +65,0.447662353515625,0.4024832844734192,,,,,,,,,,,,,,,,, +66,0.4438793063163757,0.3986069858074188,,,,,,,,,,,,,,,,, +67,0.4338978826999664,0.3998211622238159,,,,,,,,,,,,,,,,, +68,0.4354504942893982,0.3929292559623718,,,,,,,,,,,,,,,,, +69,0.4327703416347503,0.3933444023132324,,,,,,,,,,,,,,,,, +70,0.4336088895797729,0.3911222517490387,,,,,,,,,,,,,,,,, +71,0.4521145224571228,0.3870521187782287,,,,,,,,,,,,,,,,, +72,0.441720187664032,0.3850701451301574,,,,,,,,,,,,,,,,, +73,0.4295700192451477,0.3863437473773956,,,,,,,,,,,,,,,,, +74,0.4247119724750519,0.3820861876010895,,,,,,,,,,,,,,,,, +75,0.4191826581954956,0.3794330060482025,,,,,,,,,,,,,,,,, +76,0.4179046154022217,0.3774029612541199,,,,,,,,,,,,,,,,, +77,0.4114597141742706,0.3762071132659912,,,,,,,,,,,,,,,,, +78,0.4085122644901275,0.3715751767158508,,,,,,,,,,,,,,,,, +79,0.4038552939891815,0.3700215220451355,,,,,,,,,,,,,,,,, +80,0.4039793312549591,0.3683389127254486,,,,,,,,,,,,,,,,, +81,0.4004665017127991,0.3667595982551574,,,,,,,,,,,,,,,,, +82,0.3988763689994812,0.3682830929756164,,,,,,,,,,,,,,,,, +83,0.3977329432964325,0.3633780777454376,,,,,,,,,,,,,,,,, +84,0.3932426869869232,0.3615119457244873,,,,,,,,,,,,,,,,, +85,0.3881891965866089,0.3624202311038971,,,,,,,,,,,,,,,,, +86,0.3864635229110718,0.3605792820453644,,,,,,,,,,,,,,,,, +87,0.3910817205905914,0.3563437461853027,,,,,,,,,,,,,,,,, +88,0.3864680826663971,0.3559731543064117,,,,,,,,,,,,,,,,, +89,0.3826872110366821,0.3567934334278106,,,,,,,,,,,,,,,,, +90,0.3855184316635132,0.3521089255809784,,,,,,,,,,,,,,,,, +91,0.3821128308773041,0.3516249358654022,,,,,,,,,,,,,,,,, +92,0.3806934356689453,0.3511155843734741,,,,,,,,,,,,,,,,, +93,0.380172848701477,0.3484813272953033,,,,,,,,,,,,,,,,, +94,0.3793233335018158,0.346350610256195,,,,,,,,,,,,,,,,, +95,0.3758251070976257,0.3448651432991028,,,,,,,,,,,,,,,,, +96,0.3752722144126892,0.3429343998432159,,,,,,,,,,,,,,,,, +97,0.3779825866222381,0.3395202755928039,,,,,,,,,,,,,,,,, +98,0.3724572956562042,0.3385667502880096,,,,,,,,,,,,,,,,, +99,0.372528076171875,0.3403407633304596,,,,,,,,,,,,,,,,, +100,0.3694436848163605,0.3396479189395904,,,,,,,,,,,,,,,,, +500,0.0304479785263538,0.0682298094034195,,,,,,,,,,,,,,,,, +1000,0.0193224661052227,0.0504972748458385,,,,,,,,,,,,,,,,, +1500,0.0229361038655042,0.0448496267199516,,,,,,,,,,,,,,,,, +1909,,,0.987392573965418,0.0462614535726064,0.1083306340867504,0.9847544887136738,0.0551702901690756,0.1086062017457213,43793.0,0.9837648527381596,0.0583073679772985,0.1074296865913766,43793.0,482.64182305336,593.2517759799957,482.64182305336,108.7675120830536,0.0296380519866943,0.0 +2000,0.0189821179956197,0.0459201224148273,,,,,,,,,,,,,,,,, +2500,0.0437944196164608,0.0394202768802642,,,,,,,,,,,,,,,,, +3000,0.0187580976635217,0.0440383069217205,,,,,,,,,,,,,,,,, +3500,0.0269088558852672,0.0456432364881038,,,,,,,,,,,,,,,,, +3823,,,0.9884848246424988,0.0404075672711894,0.19120728049328,0.985415130918786,0.0507522517649599,0.1628767457703015,43793.0,0.984448371389417,0.0535538692609491,0.1640744205277744,43793.0,933.7262935638428,1075.9749946594238,933.7262935638428,138.99151515960693,0.0630345344543457,0.0 +4000,0.0261489786207675,0.0436869487166404,,,,,,,,,,,,,,,,, +4500,0.0243712458759546,0.048633374273777,,,,,,,,,,,,,,,,, +5000,0.0201098341494798,0.0436581410467624,,,,,,,,,,,,,,,,, +5500,0.030845059081912,0.0378625392913818,,,,,,,,,,,,,,,,, +5720,,,0.9886374956736488,0.0385616442536409,0.23909901151243,0.9857841163347286,0.0481692713999363,0.2052480629121319,43793.0,0.9848665685679128,0.0509523698471832,0.2026070270414677,43793.0,1384.9259967803955,1559.084408044815,1384.9259967803955,169.49265098571777,0.0961549282073974,0.0 +6000,0.0200516823679208,0.0395982973277568,,,,,,,,,,,,,,,,, +6500,0.0272362101823091,0.0385592393577098,,,,,,,,,,,,,,,,, +7000,0.0217510256916284,0.0349299870431423,,,,,,,,,,,,,,,,, +7500,0.0156814269721508,0.0324343517422676,,,,,,,,,,,,,,,,, +7610,,,0.9895910571669182,0.0351091206909363,0.2911635532020825,0.9861514780524714,0.0460630208773084,0.22996665595578,43793.0,0.9853134036196588,0.0486191333421493,0.2286057515900184,43793.0,1836.0931000709527,2041.9273238182068,1836.0931000709527,199.7694013118744,0.1288614273071289,0.0 +8000,0.0163229387253522,0.0340496636927127,,,,,,,,,,,,,,,,, +8500,0.0200388077646493,0.0392391718924045,,,,,,,,,,,,,,,,, +9000,0.0169167444109916,0.0355976447463035,,,,,,,,,,,,,,,,, +9500,0.0149518912658095,0.0346767902374267,,,,,,,,,,,,,,,,, +9503,,,0.9898969062830736,0.0337017683760116,0.3318149727723656,0.9865431952432138,0.0451474168280893,0.2440190009526703,43793.0,0.9856915920046428,0.0479959305792683,0.2380658957450744,43793.0,2287.257420063019,2524.9815373420715,2287.257420063019,230.2561206817627,0.1606552600860595,0.0 +10000,0.0187978371977806,0.0342521145939827,,,,,,,,,,,,,,,,, +10500,0.0207024961709976,0.0346328988671302,,,,,,,,,,,,,,,,, +11000,0.0151515388861298,0.0328004695475101,,,,,,,,,,,,,,,,, +11397,,,0.990503342524324,0.0317152428682581,0.3785622880312166,0.9866564481926616,0.044947521186217,0.2540324694597763,43793.0,0.9858512060334944,0.0476844904174728,0.2528834319834431,43793.0,2738.389086484909,3008.1850967407227,2738.389086484909,260.9218719005585,0.1932201385498047,0.0 +11500,0.0169753171503543,0.0353802666068077,,,,,,,,,,,,,,,,, +12000,0.0149312308058142,0.0309348944574594,,,,,,,,,,,,,,,,, +12500,0.0166223663836717,0.0304455142468214,,,,,,,,,,,,,,,,, +13000,0.01609468460083,0.0311038699001073,,,,,,,,,,,,,,,,, +13292,,,0.9909148422903182,0.0301297743295072,0.3991265731949852,0.9868740237514572,0.0442234500988832,0.2705426219735733,43793.0,0.9860137680787104,0.046997974396899,0.2609902563106287,43793.0,3189.6078889369965,3491.096622467041,3189.6078889369965,291.20612835884094,0.2272465229034423,0.0 +13500,0.015523262321949,0.0294452346861362,,,,,,,,,,,,,,,,, +14000,0.0190781354904174,0.0300726983696222,,,,,,,,,,,,,,,,, +14500,0.0171731561422348,0.0287043265998363,,,,,,,,,,,,,,,,, +15000,0.0162040442228317,0.0277313161641359,,,,,,,,,,,,,,,,, +15236,,,0.991443587457698,0.0282962133952696,0.4473806456356421,0.9868715882041572,0.044222387718225,0.2697428412578408,43793.0,0.9860491442750786,0.0470952062937622,0.2617299438584322,43793.0,3640.793651103973,3973.0866956710815,3640.793651103973,320.60099935531616,0.2654342651367187,0.0 +15500,0.0200170278549194,0.0333224087953567,,,,,,,,,,,,,,,,, +16000,0.0222028940916061,0.0341090820729732,,,,,,,,,,,,,,,,, +16500,0.017868785187602,0.0294320099055767,,,,,,,,,,,,,,,,, +17000,0.0178400911390781,0.0285211037844419,,,,,,,,,,,,,,,,, +17220,,,0.9920402604089243,0.026127167900357,0.4961698427316368,0.986935724283056,0.0444491568338209,0.2739300672967146,43793.0,0.9861196860952176,0.0471521398597923,0.2681840431376003,43793.0,4091.952267408371,4455.019567489624,4091.952267408371,349.97821378707886,0.2980732917785644,0.0 +17500,0.0162621568888425,0.0253394246101379,,,,,,,,,,,,,,,,, +18000,0.0196908675134181,0.0258581414818763,,,,,,,,,,,,,,,,, +18500,0.0195962488651275,0.0271452255547046,,,,,,,,,,,,,,,,, +19000,0.0195314977318048,0.0285482164472341,,,,,,,,,,,,,,,,, +19219,,,0.99225873256205,0.0252131943116308,0.5292838896308407,0.9870345669109792,0.0448523984762404,0.2790923043412023,43793.0,0.9861563257271704,0.0476955290278537,0.2709051437065942,43793.0,4543.044823646545,4936.61790394783,4543.044823646545,379.088228225708,0.3312056064605713,0.0 +19500,0.0189948994666337,0.0294256620109081,,,,,,,,,,,,,,,,, +20000,0.0174146555364131,0.0261210612952709,,,,,,,,,,,,,,,,, +20500,0.0189441628754138,0.0280141197144985,,,,,,,,,,,,,,,,, +21000,0.0227867774665355,0.0282079894095659,,,,,,,,,,,,,,,,, +21236,,,0.9925139913291284,0.0241421218834808,0.5400073620419426,0.9870051373811048,0.0451581262593809,0.281393744241427,43793.0,0.9861407433549604,0.0482126755859603,0.2743766661455674,43793.0,4994.298848628998,5418.0775537490845,4994.298848628998,407.9229867458344,0.3639898300170898,0.0 +21236,,,,,,,,,,,,,,4994.298848628998,,,,,0.0 diff --git a/logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/trial_1/meta_data_0.json b/logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/trial_1/meta_data_0.json new file mode 100644 index 00000000..be92dd5c --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/ogbg_pytorch/trial_1/meta_data_0.json @@ -0,0 +1,70 @@ +{ + "workload.activation_fn_name": "relu", + "workload.eval_batch_size": 32768, + "workload.eval_period_time_sec": 452, + "workload.latent_dim": 256, + "workload.max_allowed_runtime_sec": 11303, + "workload.num_eval_train_examples": 43793, + "workload.num_message_passing_steps": 5, + "workload.num_test_examples": 43793, + "workload.num_train_examples": 350343, + "workload.num_validation_examples": 43793, + "workload.step_hint": 52000, + "workload.target_metric_name": "mean_average_precision", + "workload.test_target_value": 0.268729, + "workload.train_mean": 0.268729, + "workload.train_stddev": 0.268729, + "workload.validation_target_value": 0.28098, + "cpu.util.avg_percent_since_last": 4.7, + "cpu.freq.current": 2200.1639999999993, + "mem.total": 359053524992, + "mem.available": 349522874368, + "mem.used": 6374772736, + "mem.percent_used": 2.7, + "mem.read_bytes_since_boot": 7810419158016, + "mem.write_bytes_since_boot": 65060746240, + "net.bytes_sent_since_boot": 32055995, + "net.bytes_recv_since_boot": 32059097, + "gpu.count": 4, + "gpu.0.compute.util": 0.0, + "gpu.0.mem.util": 0.0277587890625, + "gpu.0.mem.total": 40960.0, + "gpu.0.mem.used": 1137.0, + "gpu.0.mem.free": 39190.0, + "gpu.0.temp.current": 34.0, + "gpu.1.compute.util": 0.0, + "gpu.1.mem.util": 0.0277587890625, + "gpu.1.mem.total": 40960.0, + "gpu.1.mem.used": 1137.0, + "gpu.1.mem.free": 39190.0, + "gpu.1.temp.current": 36.0, + "gpu.2.compute.util": 0.0, + "gpu.2.mem.util": 0.0277587890625, + "gpu.2.mem.total": 40960.0, + "gpu.2.mem.used": 1137.0, + "gpu.2.mem.free": 39190.0, + "gpu.2.temp.current": 33.0, + "gpu.3.compute.util": 0.0, + "gpu.3.mem.util": 0.0277587890625, + "gpu.3.mem.total": 40960.0, + "gpu.3.mem.used": 1137.0, + "gpu.3.mem.free": 39190.0, + "gpu.3.temp.current": 35.0, + "gpu.avg.compute.util": 0.0, + "gpu.avg.mem.util": 0.0277587890625, + "gpu.avg.mem.total": 40960.0, + "gpu.avg.mem.used": 1137.0, + "gpu.avg.mem.free": 39190.0, + "gpu.avg.temp.current": 34.5, + "os_platform": "Linux-6.1.0-44-cloud-amd64-x86_64-with-glibc2.31", + "python_version": "3.11.10", + "python_compiler": "GCC 9.4.0", + "git_branch": "main", + "git_commit_hash": "b21be29be0a1573fb4f78f849aea019cdb520862", + "cpu_model_name": "Intel(R) Xeon(R) CPU @ 2.20GHz", + "cpu_count": 24, + "gpu_model_name": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_driver": "550.90.12", + "rng_seed": -1419473739 +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..ceea7962 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,39 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/accuracy,test/bleu,test/loss,test/num_examples,total_duration,train/accuracy,train/bleu,train/loss,validation/accuracy,validation/bleu,validation/loss,validation/num_examples +445.425760269165,0.0,53.060325622558594,1,0,53.060325622558594,0.0007088489919237,0.0,11.744577595723666,3003,499.1922066211701,0.0005500487022288,0.0,11.740890534578584,0.0004835649898947,0.0,11.741181138485574,3000 +647.8665144443512,0.0338814258575439,694.9455635547638,4056,0,694.9455635547638,0.5522630875602812,19.29351834035134,2.5806624397187843,3003,1346.2628633975985,0.5461362597165066,24.72222418466016,2.615983832018747,0.5493918240319401,20.598851036749902,2.567170586849512,3000 +848.3313324451447,0.0684435367584228,1336.9580190181732,8124,0,1336.9580190181732,0.6148044855034571,23.51979872328674,1.9902086601592008,3003,2191.3487401008606,0.5977578372947919,28.960536769195457,2.1133938968422497,0.6121684790021202,24.8842243333632,2.0209432462089745,3000 +1044.3916444778442,0.10451340675354,1978.963494062424,12192,0,1978.963494062424,0.6400325373307768,25.529453286545973,1.783123801638487,3003,3032.022760629654,0.6143790849673203,29.92785405711678,1.976550205612415,0.6319698453831942,26.584652851785467,1.8417314648919416,3000 +1220.5474019050598,0.1391530036926269,2621.048262119293,16265,0,2621.048262119293,0.652431584451804,26.223018677187245,1.6740173435593515,3003,3852.870769262314,0.6276181755751402,30.31384127236931,1.8615210455533937,0.644926907291912,27.520962113951,1.736397603563502,3000 +1414.761714220047,0.1748809814453125,3263.100668668747,20335,0,3263.100668668747,0.6614142118412643,26.95614184291244,1.6036165097902504,3003,4691.734410762787,0.6405194984153232,31.23142393247607,1.7526106259473613,0.6531599112224274,27.66911449034538,1.6666036378966165,3000 +1607.2352409362793,0.2161371707916259,3905.2253880500793,24407,0,3905.2253880500793,0.6718029167392946,27.845316861934634,1.5446280721631516,3003,5528.938032865524,0.6414288155870791,31.41940241037089,1.7548805759699198,0.6600042156947837,28.151750697478555,1.6223898920968125,3000 +1849.8468225002289,0.2523863315582275,4547.240791797638,28478,0,4547.240791797638,0.6753123002730812,28.228522569674404,1.5034113357736332,3003,6416.162347793579,0.646569454394195,31.908780533204727,1.696688394033231,0.6634015697263518,28.6387571484488,1.5865189985245067,3000 +2053.412314414978,0.2888314723968506,5189.318318843842,32551,0,5189.318318843842,0.6806228574748707,28.605820851693665,1.476671351170763,3003,7264.40878868103,0.6574741352642615,32.82359983605892,1.6197046035219578,0.6672576905432047,29.33604280909626,1.5561068097419746,3000 +2261.6669857501984,0.326277494430542,5831.318049192429,36623,0,5831.318049192429,0.6837952472256116,28.47953293764722,1.4489703169484631,3003,8117.268192768097,0.6546037249512426,32.60188823846328,1.64831480457122,0.6693903361396635,29.129087562232385,1.538441091864949,3000 +2536.2961554527283,0.362851619720459,6473.437443256378,40696,0,6473.437443256378,0.6858985532508279,28.86010418388668,1.4243017728487597,3003,9036.62000489235,0.6608371754979673,32.70192794968317,1.5949718509391535,0.6731596632403813,29.136297489173543,1.5138081440403717,3000 +2743.8992309570312,0.4001736640930176,7115.437336683273,44769,0,7115.437336683273,0.6869095345999652,29.056115384466224,1.413147659926791,3003,9888.822259187698,0.6736368540047972,33.463669608567024,1.5027317254943593,0.67566428190599,29.447153797348868,1.4967648029782643,3000 +2987.965854883194,0.4670050144195556,7757.446277618408,48843,0,7757.446277618408,0.690163267677648,29.252620058582394,1.3950370584219394,3003,10777.529091835022,0.6602731896896611,33.20233996811842,1.5895205249471338,0.6778093266047538,29.348687414056062,1.4871544331440405,3000 +3275.7083344459534,0.5038421154022217,8399.471651315689,52918,0,8399.471651315689,0.6934518621811632,29.445593184963908,1.3803054732438558,3003,11709.902684926989,0.6706479784121057,33.46744917235679,1.5236382675600897,0.6784416808223085,29.52817047409481,1.469598563563998,3000 +3511.548393249512,0.5425922870635986,9041.56005358696,56995,0,9041.56005358696,0.6953343791761083,29.502131903352502,1.3676087422288072,3003,12590.445948839188,0.6831654775287344,34.52225552751289,1.438460678152745,0.6807355147487322,29.87392002459043,1.462125585857584,3000 +3832.727107048034,0.5803112983703613,9683.66823744774,61073,0,9683.66823744774,0.6959967462669223,29.82273898744917,1.357591438614839,3003,13556.334055900574,0.6728275450886868,33.58934435863579,1.5079797373218524,0.6837484966088455,29.378392460263147,1.4534882549503416,3000 +4185.091699361801,0.6182959079742432,10325.675132989883,65146,0,10325.675132989883,0.6989599674626692,29.89596834098016,1.3438375167044332,3003,14553.30824804306,0.6787732503603376,34.23505056955816,1.4658047398705072,0.6824837881737362,29.899171017447795,1.4427903482287885,3000 +4498.430937290192,0.6577272415161133,10967.742772340776,69220,0,10967.742772340776,0.6989134855615595,29.55665481900165,1.337881478414967,3003,15511.32098197937,0.6870573523560507,34.462200462819816,1.4177981832262831,0.6847900212024649,28.746716660696627,1.4348646095522684,3000 +4838.17728638649,0.6972668170928955,11609.769958019257,73301,0,11609.769958019257,0.7004822497240137,28.9335588426571,1.3271995380861077,3003,16495.714752435684,0.6781031888779538,34.477809736742664,1.4760303899277158,0.686451500911334,28.023441115401464,1.4277885968555877,3000 +5137.922399997711,0.7374632358551025,12251.764683485031,77386,0,12251.764683485031,0.7026552786008948,30.07638390589513,1.3144139285921794,3003,17440.068254709244,0.6885287531454164,35.03851466280576,1.4049029177343768,0.6878402003694932,30.26158475825712,1.4182358634734846,3000 +5425.737843036652,0.7771332263946533,12893.859102725984,81473,0,12893.859102725984,0.7050723374586021,29.86448916788208,1.3048747893788857,3003,18372.59902620316,0.6918920154002582,35.61374162031927,1.3938208320480745,0.6870466578219737,29.677615166629305,1.4101627885891062,3000 +5774.228710174561,0.8166906833648682,13535.897929668428,85560,0,13535.897929668428,0.7047237232002789,30.09348926656585,1.2981343690081923,3003,19365.742505788803,0.6885686927180703,34.92511946075621,1.4019506491952032,0.6890925097022975,28.31170686241403,1.4029911671894957,3000 +6005.925210952759,0.8602900505065918,14178.007321834564,89648,0,14178.007321834564,0.7057695659752484,30.257084271714383,1.3033401603625587,3003,20242.164640188217,0.6970119567453876,35.79026020455463,1.347744750348643,0.6884353572801329,30.20290091938968,1.408247805358892,3000 +6226.633292675018,0.8993208408355713,14819.999969244003,93735,0,14819.999969244003,0.7076753239207484,30.66273337214052,1.2846936552204986,3003,21107.47658610344,0.6938509699974864,35.663830096058305,1.373772329079359,0.6909895723549615,30.513111798349964,1.393101139477502,3000 +6583.471929550171,0.9379317760467528,15462.022831916807,97824,0,15462.022831916807,0.7093835337865319,30.381134391662084,1.2811286205043286,3003,22108.956053495407,0.6972819389292072,35.71535766677376,1.3497797352115568,0.6925146619384757,29.095635009310403,1.390379052181622,3000 +6867.136032581329,0.9784302711486816,16103.99957895279,101911,0,16103.99957895279,0.7086514438440532,30.30002914573537,1.2787330776828771,3003,23037.213791370392,0.6997586999531124,36.22656518179386,1.336101624772709,0.6933578008952151,30.279620503488072,1.3865799866089694,3000 +7131.977272987366,1.0208284854888916,16746.064346313477,105999,0,16746.064346313477,0.7098018708965197,30.5515793034812,1.27362170269014,3003,23946.72591495514,0.7005885488132634,36.09823027073485,1.3351153460701042,0.6935685856344,30.316039877116665,1.3848594174281783,3000 +7385.803707122803,1.0621931552886963,17388.03769659996,110087,0,17388.03769659996,0.7104293765615014,30.43325747213432,1.271752258729882,3003,24845.13564896584,0.7035749922581978,36.38694155896813,1.309232314282765,0.6942009398519547,30.652193336253045,1.3826231230858883,3000 +7647.055639743805,1.104595184326172,18030.13545131684,114173,0,18030.13545131684,0.710499099413166,30.540808370113545,1.2717316505432572,3003,25751.09917759896,0.7029496215087444,36.05714637756056,1.3174546640393583,0.693680177555145,30.78495669829378,1.3834639372109458,3000 +7935.464534521103,1.14502215385437,18672.25456643105,118260,0,18672.25456643105,0.7104874789378886,30.5197138290683,1.270655667160537,3003,26684.23622441292,0.7059680927383081,36.6821103814912,1.3004796195497454,0.6938413658851099,30.666652184073417,1.381956768204982,3000 +8208.439351797104,1.19000506401062,19314.32937502861,122346,0,19314.32937502861,0.7104061356109465,30.45658956250633,1.2707375551972575,3003,27601.87896633148,0.6997066171583122,36.64013427257552,1.3324967338238327,0.6938537649874149,30.657045900752426,1.382162225204895,3000 +8474.484276294708,1.2323086261749268,19956.34680557251,126432,0,19956.34680557251,0.7104874789378886,30.44486588346862,1.270744908779269,3003,28512.538029670715,0.7062779241608431,36.67276794079318,1.2974285606885096,0.6939033613966349,30.73521655991802,1.3826583830330683,3000 +8739.491944551468,1.2747008800506592,20598.37356328964,130517,0,20598.37356328964,0.7105572017895532,30.34159563464225,1.271061203590727,3003,29422.16049194336,0.7066352230057698,36.1449996008607,1.2944492899418445,0.6938537649874149,30.70360554311707,1.3822447567296128,3000 +8997.632767438889,1.3170082569122314,21240.45601129532,134601,0,21240.45601129532,0.7101504851548428,30.57973386492272,1.2720226163500088,3003,30324.969521284103,0.7052066596013812,36.55484407583282,1.303047331715155,0.6939033613966349,30.8441882006884,1.382166681132286,3000 +9220.341248989103,1.3608474731445312,21882.56513190269,138684,0,21882.56513190269,0.7096275637673581,30.415895458234928,1.2750479344605194,3003,31192.38531279564,0.7082950367647058,36.50281435992577,1.2883000990923714,0.6930230251329804,30.805347272254515,1.385253767002269,3000 +9463.840975999832,1.406548023223877,22524.6876103878,142760,0,22524.6876103878,0.709174365231538,30.39403426658349,1.2770021715763176,3003,32080.60576915741,0.7007713844923147,36.41966519680081,1.3295150991371922,0.6934693928159601,30.73410874423248,1.3867799221336374,3000 +9722.284867286682,1.4494123458862305,23166.677849292755,146840,0,23166.677849292755,0.7088489919237697,30.26458936712677,1.2799526828772296,3003,32983.62984442711,0.7025382336456082,36.45069613420243,1.3191507633860655,0.6923782718131207,30.375924389895964,1.3888317798291403,3000 +9979.016785621643,1.4916207790374756,23808.78427028656,150920,0,23808.78427028656,0.7084887571901691,30.23243484427275,1.286095738190692,3003,33885.05792140961,0.7043234266735383,36.57719162324027,1.307740741138407,0.6922666798923758,30.715556801482006,1.3949919057110265,3000 diff --git a/logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/trial_1/events.out.tfevents.1789549050.1803622dc733.37.0.v2 b/logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/trial_1/events.out.tfevents.1789549050.1803622dc733.37.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..23205bf6657e09a78b87dc0760cd0a50429c9ed9 GIT binary patch literal 134694 zcmbTl6@<(ge+y>QYwllC6O%&Nr*xv2~qZJ6^bm8Y$1fA zNTDbRk?LM^{N}vRJMSd!zxRHhdp`5dnR#ZKGX}^={`2&Kah|-4jOWT7ho22>QE!%u zwZnogLl)aDSk$Mhg{DHgMRp5Zom^%+I=OeTadNTi;NvxnjD0HsLv ziSo8`xsv~LErkvWigO&DW?4JVbX{cU{9nYc|1m^t_*!GCmVd`?G&_6iD; z9CCX^KkJLz<6`UHUXWopo|Q4n(a!C^&mr*iJGAzG|KFd(YWSxD`S@vkhQ2}_d9C- zj8yIgt2k7SwhVKL&OGo%lFuMy|Zp1wK*y$mVGTA!DDb zAL;WuM0xxBSMt_Z_z+zM9fgL@E_QbFoquiqnKn*t3l{yi2!fBJ?b@Srm)}WAwz-o;+hcYDA5rb?G5K5eN>J!O zi<2m73eQzH`%{Xy3QOZbQTmFXuBKw{sDZsfu`i#G5Ld_;!>ZP8_wytug1cOkC~656 z2OCq0fpU#qL9wSdKh0K@(`ymAWO)K89u4Fp#1+PmF425q`+@mz(itrd^-IAZKreaR+M?6NwY48!^ ziY!zd)_1Q9inJj=C5pO2g=;|%RCI_s9SUnPdo(}IRtS!?E&Xb|g|+xLk&h5p$l&LA z>LfD*6uxe9Ke9y&)eVt}sT$`EiA3K;hb@o8)uU6DpbvGsNlnZmr}& zk=KKtW-A1HRNhIm+MuW$z(qa@ShZnjbC-&RrIswSH2>wuzL0zb`G2==IucxOFOBqZ?> z;tEexX!n*k0YyT#l|<1nKqu3l*i~y5RKAOO}S9xZ04PuBIX(B5x!pOjY>^afLsA4#!~i&7d$gbeAX^ z3l;0#b#XecV0#)UeEag#)l}5*9fu8M;$S{PToHtyqrLj>%b>_J_mU`7go?vGI^%TP zRvvhLF?2dVT}?&6HQ8gZ7Vm8N2ysO?evUV98r1~Fi-1iMMH8XobweGTp5CbQ5Lk%A731n@1Ey`DwPIj9!aX-=~>@;&nGZ zLR^uEpTjSFTzgP-8FyZyP!%dhFY18PZui@@21VOx{B$)H%RS3VK=ET1A0e(t#?KKm zb7l!B1}sUEC|U>=&%D~>^x2;0ia>F}lb>cQ1n2JT^}2&W@zRHn5Lcw)=Qy-wR3<1? zPNzx~Erp7QQ#Engcjnf+pg4blpJpord(@1YnlYg8xW-3_E7I|E4E|_!4ir6KJeMfc zgo<898aREeykQeiG4#%D+JAh(~YMw;VN~oARpdC(U zy)nB0ib?JGX}01!{q6pGVZd=vWN7md;)=KUIleqTbrTeAOp7Io)W}*DtdlsgVR1WqYr^1a6LcG zRtVOj-SB!jposV9Bg7S@_&JQP1O|d4>S*0^?oN)nP!Sy88mE7JIbaS7t+V_zTOnAB zgIji71BGP*A0e)g!9Pan-l)_Eh4t&k5=C2~!Y;QJDiT%%+yce40)CpU_(tzhCQHuU z0Y&{Ue1y0{78N>OGV6e1wsISZqMcB&@~s*w))Yo{0mb4L{4`r3Sc_xD#YaK$z8xPS zuBeL&g)N>T_!lGzU+$iFwtEo6R{M1QM$baV}#1+j^@pVIIUr?B}m?Tkj6e?spHAjWL$CGEE zh|}Pw*@`dp9(6F>;u|Qg>hKZb3N=)W&Y67=6k&sANfcT_#kU8|P!WHzemN-ajpV1< ziV|A!S;ZFb{Hu=VBg7TzsOZtXn*t~ny3Lm;Itdj~N1LKzn!BkkC}LOe)74bCG?4EM zYw>U`A0e*LM1}j#5!XQB6uv~F&=xAj$x{lQY1gos|2)J`vlXA|wJ?|x`3w|8Pw^4r z3N2J*Es59(ik>glNEA9kh0R?RT#IEdzRZWUu+QVC*@|LXaX_{Ad{D?0@e$&R&Zuab z`Lq=%V(M>^C^`!jOZ+Lt^BwOTLGiE|Kh0JY(F)T?r%XUGK%I{eSLmZcCGS;#P<%2C zktlS93NwesxE8%M!(V|S$ef>MD?ZVRMt!^@LGgYHA0e*jhKkwG;=h36u-5^JqKi=R zJy#hOb{mwwfWmGgKh0JY(u%u^u}>!ju*K0;j44;9zv*BJtelix2$6#7DiOlBioi#{e5M?kT( zX8eECY()XB=wIv5Pgn~_1wKMtVT_85cfIlM!BwprlFwlvR8(AVh>AfC>>GkY!+@V= zEAnZD(vxgWP+0ZmBg7ShQL$mc?CG!;{cRse6kUai$9pNo6V)-MpeS(Ur`d}4v?5QX zb$?JKE#f1@6{e_YQ@{6cPz(ynlqd{^3hm1ca4j|!ACduue>gu~O~sKHD>s7TWfUJF zt}sJ|(nsrNpiqDKR-))8R2Ugiim0`#j)CIIbAFnwcuTK^!2^qfps1U}M~ExNqT+eV z?qE>3CzMDO-Gz#AGL&LresT*?lqB)fY{eT|k+QN3kF*=^@)6<+>QUCHTWWZ|m{3|N zQS=Zhh8(Gnd(_~TJ*I=gS4O6SyMxk0yPAq1t?sp8EoRE|5#ox;xEA9o6<@$wXy_?Q zK1WZXVx2RkXdF2<0~858`DwP|HN6(GM{*Z};-e8CA+DH?id*Vh#-K>GZYojq5-NI` zQVOlEEfhgR%FwP>pdEHf#QJ+A0e)=LdAds2Y-NKV`N*2qPI{HYf=yQsFYoX zL7>o$=BL?;ELve#{-X~lrpECR;tE?-l&Zcf1clS5&Jsl*p`!L{B~;`Dj+qOJ*+2Mc zwxZR8Z}{sE?;~C>K+&g`?tdRZTw#xjt@7Kqg2LFamn5XGP|+xiQW!V6JscEW`tj3j z#liWXP%-m+mK!KE2lEl)3P)4~Dz(SI>%Bf?B2n}cDoWN;iu*f<;9rUdoaU$5im~J0 zpyJWh6*ob#{Q@5$u5d=ht~wE;U@bH*Q+ZPpH-J@}TxPqP(eZJwbb zVOFsMJxWELpJpq(-EN`6NJjHL zC^WVB2yw*{RP4^X5ebT<$#xQjkx;R(C8fAL)9ffHJgxa@w!+@$Ix2!Q7B2(Ea|b>` zT(KM#nRS}u)AGxFoFxilp(1T5r8uvfunH8@xAN0$#Y%@*RG2?B!Dn*j1@jT&3Qtt% zSO?-BWqoCtM8Wu#ol}YeelPCl^xX)G0eARmwxV$DQB;&%OKlHparF@&A+GR3#l4g% z$*@N`)Lbu73=*z|xeujSw=*XP6#W(XX|`fyz#&xBySf+u#;Vhpj}TXQqr!KUY)?=) z8f}v(OoWQ!7xK6kIYWvHK%r&IPqP&R>K{bKj`@QuU@d~i@)6>Sb*RuCdv`V{?A>=u z6oZ8d_X4=6<-`FMP$JEhoG<- z$WOBs-q~AGadcI-EhxN)^AX~T&8SdatYZp_33F2<3R9tCdpk1)2yul!DoV1Itw5o$|B*y7T&T!yOesv3h3*DLkK_C_TXAE77p}!i zvxmb#(K&{X5LawN#j8i9PM}E0&XOob2o+6A>*A5NUwp<2P)vHyPqP&p(YGGiCRFsAPOU}0<9j1em^<*(Y=!sHWvIvqu*cu)4O_@Zh%16n z(e%FBd{~Qf2WnPw_hLo~6-mP>#jbR()1Vk|f}dt9Os+ejB56fE{2S|{SUy5r5rT@A zPhN$C!sJywiDI-+5f(rxE;s1#1{9O>`DwPo^1?h+=sTKEgYU(G&wPZqA`}&+$8{_~ z(NbGgq8KAogho&bHP5T~>nc4%ewwXNs=WvmllJJI2Ze24K0;g(hKd0fTekznjoF$K z#aN-@EWJksf1C6J)}o6uKh0L`nCO6tcf0OA0L9uRe1y0n92K5QF?iOybzV=RFc&IP zcFEz9_Q8*D<3KUvDnHFuxEnd5BK%G_d{*tnEj~hAu@4nfw>kR5TKIqKBT-lg6}K-@ zicKr?K7-e*^1~1cBr`1r-v-8#lE_Hgt#IC6<0T&Q3r)v-=UJvF;1v>+K>8P z?5L5u78Dx>^V4iark*7#bPn0z^R#|ue1y2-AS&imx()_~y_>m2F$@$dXgX0?{iYY>cksq~3?atoR2o##J{4`s!p@3S8ejndI z1cgo_A0e(dg^Cf&RTjWntau+NQA`yocD|w3LcYufzumn)^V4j_o#&HrEhbO<@(2{` z%lHU!#TiuG?LU7lD7I>aOBB_2rT8((%o-Fa&ipi6(TF~Gd+S~C1jXqke1y0n4i(XZ zrsETL&kn~(6f=Yhzpd0B_1JYn7AQuX=BL?;VcV&-uyKk12nzoTe1y2-JSqmQls^S) zkza9HqOcSy%pxeo%!=Xo?7=m;EB{Ti6-^IW)E(yp|}e zg^J?&)LKkY^|=U&xLf=*TXA9G1XSEqjr4{+>cT@lLR@hL6<^Yn++Zy}$`wi!vxJIa zx|Cv{ieCgMTpRM!Y=zU#iKwWrs_qYpsOEfxxZ)Zr`pevz0SYbC9}cOsyX99ao4xD%+6nLZL}3CvTst~i(ZSL zdxK(s06)!EC`V35MR{2Yz7OQ~E@}BD=av$Jvfo^phZ^?~3~M`2A|PF_ZiX<=?E5W{5+6;2ZGj2ER!J$c*osG&smx z#@m#)8eb%>b{d^h(bIsAY|2-Gc^HXeo=KK2T5hT z-=x%G2}1VNwx|Gz>J8~Q8zOk%%h>aP13<>ylV*rQWPTvz+0e%w0kW=TjwJ`V#&{vc z!)G%PvTKgT6M!gcO2^p{!82b5G3%EDq?@iZLlk1HhA$e`ysF{>cIBHIDP{ua z2q`|K6$6lzZqji!WYf6^c=YXZSxFHftNTkcL?MY&aYe$+f+hpR#c*&42MKQ@Jp8VF z)DU-%>P8Y)SEY3A-~qqS zOWaGUPZ3J!(hYld4PJ%%wnUm%)r*YRnQ0=f#qI({NNV&`Jc$&#q(q=_ z!??tT@YYx*+3x(J{nM+A^8>t_!aH{j6s3vc?=o&>T@Sa`rI4?xuww@*OLMp#TktaD zfKKgkJI#3>>;KN$ z{v~y$d{{?mmhgSX-toG48uC`3GZwyL9rdL1RlLyH!88hkUMP9G4*mT>T)Lk-An(#$ z8dTjk8Vf$&i2J8XjDM$UH7nUJzrUX}_0O*~cKg;AKjmdvzg&~wc_-Jl50_tVBF(GT zJB^KRDD;PFwc%=yCUDHwA1RF#J=A%wz(d`>L$|Ep4c93a(v-h@tFc3QI~O?oD{afo z;*Oeu8p6Z>iZPvV4_6DEwg>j`*bwPB`|zKpavslPD|R(Ehi$TEzcfR1n}qw}HfhuB z&JciH-F2{pgZMChI~@5?9U&cMwC@3=C_*~UhLq&&MTq@E=Kz4TKPJr(g&2n-WZ9XV z<^btg;Z!&1*Hzt5#!pti?P-CK19v``0A!4uGe6FTs2$pmkj-~I0s&ILp)^Ajl63?j zZ@0yk0_17iK587Ki18c4@Q11hu^4E&9w0ZgrQ>XfN!?utY0`LZOMpx;lxB!RJTD;R z`0_I)05O>$qt8Kd7(WVYv$rWi%*UK42gm?R={OrQKqd?!-`jSN21xxm(hO0E`YnWL zYex?P$mNa6CLAQ4@hhJ9iOmsmr1`fR0LcuLjJAI%WZkO7%*9AjJXHRAb*}S@( z$iwEfchsnt(7sLAOGEzDz8lA?p(~-=B^Bt*T{cV8s%qcuCe=XPp6PyLAgL@+Jc$(M zDBnQgY#slu(7xO3mL~mE`<__O>oBzM8WGY&@%Ei&(+ani^O2PYV8^a|M4H3x*n;-0 zb*Ta~WEE2JW`XW0RtC9+{Iue63mvX|Sq9p7>N#n~U$*bqBkear`#zZ{%@VfnsMe8K zM;#n2j>1=LVX}0-iuN7&r7nI~XJ*&Uh4!6!R~l5^_AU5$-bLO6pnV5FlBWK-eRpp% z9=S&bue|~7J1|3I9K<0;36BI$qfYIDZ;8r4={WoFZydH5Pskq26FNU*`TL~7u!EorQ>W!VYnMYK2N=90gyf# z(hN~ZxDrC9*=uD1WXh%TbPf{Ac;I^7F$ILY9CQNjeQ!>dj~H&-JhAZ+?(fK;YR zGejZIDhL@g$3+t$dlPOHa1ejS6SCc&$|B^#-K4d!YjwFH9cM$n47Wi@vjBg*j2L=P znjs1?RzpZit*)&BlAr7^6a4FX$eZyd<7tlA4YF$gwi&p@JC2QWbzi znM*T7Az3;IsVv-gA0RVLG_^QL0pro9?9^JgBIoRPHh~qnVkRADL;7~4{tBGr{NyJ< zOeRP(L?O|J2)Q?SNGw2}c<1-!AX$uORX%tqBBal<*iiuKx>-8ThQw=+!(Ge$=7RocHT7~^K*p_jRtC9+Ec)HXEp+_LiZW>5O|D5Z{<3|aS9_WV z?fXuOG)vgNA6b6GI=b#_Q(yRsEqNfFucCd=@a&KEaejq)JhbofbZJm^+qdB3C0}q? zhxVQKQkweb_U+^6h1|+|->yLWj?0ziRj++3KF0Rl{)%sRIMk_qltzjl>IUGU&LP61 z5Zd?oFVd91Yv0Yc9FB&=|J|#_p4?G0jqym9-=aEr_&@bwXnWYhmF`H#*@yrA{l?(o zzi8uiJp5mJB+U@rCSy%-n-~oG7z_}3zsNuil58iuiQg)Ng0xWfXaJC>fzoj{WPBP0 zNelQm3Kng{6tNe-X0GU}&I?jgFUPwVk zc&Ol$CpDT%GejXp76`fO^*s+DEj_j3IY{(;A!N|*+PG_}PwDm&AVC|X<7~*;@*%iu zO%Khl11qv;n>0fdk~kG1R`ZM}0A%x&{CgZEVxbVCokv|p{D_T70m!~t(s4FKW`2K! zxGwFh4UoXO(hO0E|15;q#m|cZ$hXPIUvrQkHz6e0n7Zk>`x_y| zuiF#Phmx}9!oPsA^I)|30HDj2oR4ey%a-!UD$c95JK{;sS7*v1-^(urJyL6DU5O+1Mde$>uD;S*;!6=>fkG8z1${ZsoM>T-VwwC@da(nRt0 zeWi9sG|qTjI2v~BdG)0^+>R}1--bgUV}{Gv%~p`HrU@&9+(H+V@8cF4(P`mcXy0el zq#1wNzRgER&xQ7VO+%U`Y~R;YHLx^S)ZeBJUoi_E>3kLKyYP-P28{?Qx(x06ih(q! zy6s!=@tj`XxdQF`W-n>#pWCMn!DYTeUt zoI7Gf%(G7beWvK*R#|4?{2l(``?O(1i0Ec{>xr8sR3`W-SoU73JCn1#^=4RJG^PGj z)%sq7BUpOeB<9(c;*id0c~|oCF<9n2AVNefsorQw_?j>dEGf-yyKt5aABIKt&{sV4 z_&&Ek43_(CiFvlge4id#d~Al+1B;I~5h7}d^FhnGGo3zz#bd+Pb(|&Dk71d$=T zE$mqcmdI_yJlj%GpDG!9O*QU;dMAQ=QhZal2W&6QW%W6^xXSwFjuo&+Bj+R@G zXPpAe0taHAZMovp7T2Zwlb##EGS-C%5w-aEqow~u>v>?&vQazES>gg2mbd`wCS1Xx z#AaaW=Sa-6Ep@`$p=Ex+*(|W!Uqpn6TATyXa%-wHp4xP`zq-s>js`I-gUzUQQSKkt z6)fGthQ(4rGEC-sn>0K0h95h94&#Rc_zw87Z8XWqD`}$?aX#udW zzx5&J+2@49wp6#R==^XQY?u98i4f84q7{zYMKFBaQ`h@+U|L~ z3M|X!67y`!&)QUndaB&o2P~6ai4ajsX(U=U9$t72EMbZs6F5uJ5r)Oc@-tdGz7L-Z zmbog#JlkS8pbM_cfXPa7V0ocNgos+Qj-ch`m(&cf*ys#&;4C>O7?u`msQY)tc_nym zi0DSlvn|(WQr8?mANfoJORN4wh^Qs`1X{KQ-dqosb_Z&&;4JB<8I}#(shiD9V!L&N zb%{Gp%(E@auQkJUNwx3w6)Zzyi4ajs^l7wQl+kyh2-6Cb7ICSrrmBU${T&%LEAMd7#M}c86Q?5h9 zo?$>FiS+ajd#Ij1r~W?tbJOcy>=d$`6-_O}a;Bl2{S$h+oH3E|r=BkVRSRh$Pu7=0 zx&APeh^wlnFFkw@qwdyk76egOMpY3-B5(G{Mx^(e-g?l}jV2ON|J2iy?zjd+PdAuB zM2h$HVP!hV3<~V90(Nl=TOx?t#RWZmW=s->bl5g)2ZS`8#|$C2QunJba4VTy)M*br zy{`)q@|Qi`-)CS$=;?Axi7;VL-_>{#w(INnRd&JG?YJiqP(@E)ed8!5O>d}B1U+4E zJ&{!1o-X)+S54$5K~K-$M8y8Nr?1aT#-2WRU)P?{)5mQm0;|>28;+yiN!YRNjfZgT zYq^`q6g~DGzlg`a!>PySz@J#mB8Zs3>*>0xm0@sB*tqb*Tkbe|<`UzaV6%bhgR@IN z;;Wj9J&1YsIbnFuv3RZ?I=uw%bxvJFgothz`%Ac8oOX|x0hR}S=Kka?QCAt3%vh>J z)sDVZ8@{&{Cd543@*%wiS|+JKD}^<_K9&d(wT!)rme(yyUV|mqSi529ulrD;HyD;q zhPikQiaP0I2bRtwh7%W$Z6Z34#;QOu6()xVYbg-nD6Ct9Oy7$oXW_Cm~uv`pZHj=ZHK44h7 z+^0HES%watEL9H^^K6U4wbp2PopZW9SQeckLPRY^56}{rVSWQFLko;&a+ZR}49ifB z5BPoQGv@zKf&IQM^EhcnuKd6+#5~*bs6-ttb;@dngJnxO5h7|ye~gySDcPA|d16-A zm9u0$V_3?bQTH1jo~RcOmV^n!JlnEu3AHXIQ47X`WuGMxB5Fx|hL(w*N}r)PELeB^ z3TL_aqRO5gu$Fopr11E?Oz7$3{fQ)zp59dVDXz2Goi<}3!Fe2qoJ^e%)b$57VtA^*HTt7cR#8uVP>o>lIzNt-W{Di1s$Et`Tk-BPc5b69n zMF)Di?pY%0pL%+Hulb9hr@O`zk>WjlaCT>8K5E@l9(M7bSBW5Q7Z>#OWT!+7>7@P? zkHuY5m?7j=l1-zoX1?pDOoyJXd!GpT%buS5>V+=!bp3Q9OxV*usBXq~?PmDt5q#ZB zUlIXT^mMb;$(R&3b4Y*a=_7NAr0Vu`!3W$nNM!@`^okEe?4NtOYTrEU>6aI_(}A9T zse}luR!>jvwHlA(2fp+e4adHXKZ#7yW8d-@c29@7NRNGg*V8Y|yLKDS z2{)%)yviL%)Bb-PZL82mLDtd9)pd|U-5Sa=?PLvei*R3$HvWaNfXT&_) z5}ZOklW=iIy_#TI_?ie2wU~ZH%kGzR@JC1*Yn?lAmY`yWrEn2-{S;^9Yy%dv&BQ$0 zl59wI+YJN!dxGWt4kARZR=0W zvn_K%+T(F-{@A;Xz%qI$5h7}7{uM3Z**ofkWo=;TD9+;fgJE$g%tlLI%|yIUIBPdC z&$c`p+z~CC)mCf)OZY(|MARbl11<3-yR5*n=6jqqXDKOXSdu(4(9-|d)e_jpCf7VC zoo8Eo7gFb!6W$%#fTflq5h7}NTaK3NbsIf_5~MQa;ttLdSyAORN1w6O4fIx_T?^rA z;gT(pBy!EM%!0ai9(&vtf5O(zk)1*oP0NwB@QuffMzqU;YYsP8BIQrl9JB9h0Eja zZYA4J^3HJ0vEU#P@|V{fImLzh;hJO2F(OQO&EcoL53dybu8zk)#q~c+1XOX&F{IXO zOp1}q&4FuRxjQKH$FA&ziwC$H^;1?4PeWf+x!0=Nzdw!VIoCTBQ(y)w<@0 z%MQTfxc<4Oxp3?&zE5O|9{WaD;IS|DS}O&(=4kemi21u~juu8#?>Ju)62%=y->$AH zyyjS}ornAS3!RPmu&=)?C+6Ab1UDyp+%EpJ7c_+JvZKx}>4y;AE^lRq!K-FIUwWDk z=eMCt#>aD(jI|7l_f_g#ol>rNA1t$06Z34#!Ee-kTmO3MFTkSXON5A8GHRmbd7E+g zLe{%i8mDrW)C~-aRz7tPtm`V3dSKaqo0w-?(#}!Oy<9Tv)E6x2X+(&qB~=zJqsz~C zfOWBX@Z>dTxweU6Np+;IIb!yFwF1lY3}T*bS!+jJ8I~3%FY&wI?R3is zS#Pj76~FDuSps)6EK3fjqouC;CNHo&DI@0DmSf(%ab4sS>g@!}H`#a64~USXYJJjsRz=}ueg!{J^k@yB1xpDmz6!hZ4+|nfh(jKSg}*c zayG%fCie6*`$AVkPaimkNcmGw*PcN=ky2~1OIIk@TNV&;RrU1!`Pb05bMira1)Y^! z6;UK|X5Dw#K<9NI{0@5hz!gN)KlStjYaI=sr?>GYBE@@p_j-EBY_4ND5q9ww8;Kxp z7Z>#O4EJ*wQd3JE|7o_7+n6EbR%-J2HEyM%cXvymr)LKfA%EG^U!GmD6ngrTa3W0D z(>L~s!Pe@p?U4&#w;54HKovc`{bA}|hEB()2SZQKJx(N5x2Fp};6j;+J)x(^o+V=c z+|!Rvt%u^ymaFzbPd^?{1Xio3r`Lj&Z3H{}|UDSGT{&;XBpM$uYX(9^AM z5ix())7Ka}cY||+UX$-0+;P-rFXNnW%$NEN&iQ8r?codavlTJVJ||pSN4*hJzU*Qg zY?nbDh!D~3;?oef%WM0qN?=hteRMNt`5D2m#9B~&@JiICK43WzPt3C|W0q3?a@$mo z3**2tIf)1nwJcXg%d5N<%fK>ispCG*QWV9o7&N4wulRm$g#%b-c@gt$ORH^E?~ogw zXaJVJ8;KB6i?a$^EL)G02g?FO{TR-YbChB6uSZ>ToQ`>q&klR_Bj(wbh5csXcW=wt zX3xPQH-rcgwb(XA%leylx`O55s4q7;OL{cJ^0*FlKTqGx(h$}~dlE6vwu~I#7cDiO zM)e2Fo0&w2sAY0n?G2WLN*6LY%k47^i%|}BU0b?oZfmg2YC_DjEz8aiL(4RK zwPj$b+nNXwwV1U)OR&6YTd!&mE49m^B)Vf&Qork|oNoYmPvn?r6u4oya zHZKw^&K-ylQA@YhXc-V)O+0(;2QxCcrE`2%>8g|2XM3P8PSJ1kR z-Rx9{*)1SNzlfbemb2@3sh*zs>cJK0>6YJ!lt1-!gYnc8DM#`TuYsQ4x`K$Ss;9eO zyNtf_2yY9BYF_)cbbHfLBvPXp)zb^p5-g#orzsIp|J2j>HMjZo&|Nk0W zWG>e#eF|UK$t{T>ZWkBy^tEGRFeGmKoFWKW+Kw4QZl#skx!BWF>ST|Co*vtY2>HvN zK4O-RIrMaQeIiWQ(>+I~U~3IEwi^aL-K-Z8P(@E)*0mnq3^qOFs|P(jWB`#=-JUM^ zfE$1M4}+c_HiU@%b5Bq3ZjE9`JHK7f(`TCzfz|5i(KpWFv9IRvY&?$t97kk|9{WnE zV_)O*{-w~<$4w(*{;sEI*uB08=Y+xg3ukc0(WpemIYGWZb$*MxKMkLQwmnMBv(E{i z22pP++)*6uf#-y?M2P5i(Nf2A!ibQ2=fIL~ePAJH3BAU!jP{^jHRWBu75+oj^7Dv! zwq=bi^?bT@(d*7&DRU)4L@jFV&|giJ|1JmFb)GUUW zXIo?@QvWhcfJagXu$;e0gos)iYNBPsxU+$oZ#8Zy;c4d;XSYDTZ3h^6%it8DeZ_BH4VQsu*e@en#frS9x^N+4^#V?j((B`tV`ud zVxDaoH;4M2&59rWo`7Zdc_KvAQqT!47G??7VEJU@n8sPMo-iybeJM-#P@B(Sx$8*G zvn?kxY;hl(^Ic{ESQfewA)=Nn9klq2e7PGeDb8Kra+b8`49nrHXSk1*j#&@^miNnu zdA6l)R~NMGzdaYvReQaO5K&8-E?SZc*Qh~ph`E^FA^g`hM|NhFJ>A`vDzk~Mt!_iZ zj!q_$M0)y~DT%mk!WNszLyB%HJB2J~x1ZI<`{~d3mUe=kZt;Xj`BP7ySW40Q#rFw? zay|G35l5G6@|oK5?U&GZ?|{YvhI`nHey2WG!1rWVt*6p`QyM~Ur4KG| zaVw>cdJqjgJ+SPVG~_RP`pCj*7oew?)qE}uWAyZCFR-<$Yk$JaVjD#wpo*S8VoeL| z=@0WCZG)b!uS_IWyQkA1P%+$B4SKqVDiQnVo^G>J2gMgVC>uaeS87WHR;#D`zrKyf zzE*QIPr`9LU5m&RJ@#dH!DC;(@+CLu>4Wu&n7`}k{vnC@`z!sg?ew|hX!1+OIlq9x`Wa%LZIPMbh3nF}VJPmebK{8+QA@NTTE>kCTML$t9Vg7= zED`w(%fbHCJ?KqcP3nN9g#j_owgh-ky<=O)y7{m!TY3{AqLzs6XmPV@c?>M~o;O>? zS%L}~mJ|D_YmOF|&(8o$-#5fO+p?)Eb+6Mspd6n^GWbY@h+2YrqUGbP9go3sPVMP7 z&f@o(VYzjd`deCMpRVp;nbd)pXIo~>qW%He;rd0-U|ojm5h0=$zusuMuQT%|SSl@d z9po&Y-x!ucL+X8VlVbBs!SZPiG0(PaRQAE|OI?LbKd`iOCPG9lo_*1hdCn^YEQZad zp64tjKNyyG3#fg}>iLgku#8hD=Ghjf&nwYlyeO(1ELxq25K+sb{%9$=5?};9eR7RD zwfFw&>9@+O?CFQ)sQc;tG-q6dhCNz|ND}Gk4sO(MMw@nA)(TRdH)f}h<*e&HS?uXo z>-39(p8mQ8k@BaW{?1z$57tfc-^oI`ZrhHCtE#7~OuL9t!_|N2LR7O(RYZ}l; z>9XBgx(~{q$rbB8c0?1wB2${}hI_ zju?c0bG|W@8A5I)%i7ddQnk#z06pDsG!gQbJ^gW~b+4hP+f5+Cggt%Rykcyv1NKkZ z0be)W8AL!8J-zj8E$rzXeNME6p8n2;NUCm67kt36r6ZN0r(d5-#QwRbFS^tR#n!cy zjG(6{EhGY~)zeK|7UHpQ%d^J{aO``!l*kl4_T3tQ$3E{E6$9w$!&ehAf7jEy1#CC~ z=L91^J4Nm|`nG~`PUt?II=?M;_r~X-Gj|a4>~lh5N2+(!YY{68+hz71B1Ck%*c;(? zNx$q83YG!ynzi998D6!7&)VA#qJDq1&%PnPFSAZDG0(QlQKGLoQe*Mi;dVcW5K+tY zfoL)9I&D2z?$mtLm9wO-V_4p&QR|X6#=94st1A?VdA7xK*#_KS=QzmVGu^eD5Fw(L zu_kDley{UFuoxNy4&^M@d>NMb(Nyo4anm#wEM9$xdA6nX>2+vPxL)xN_OTy>h!9cB z;2~&vpjt8&EM^PFPvtCen;Dk1Nz~t;UbyPwJ?Kg9#5~(F-NPF#Mjid|-Jd?Ih!9ar zuVHA(UN#lqTNS)ZWdUb7x{YBe=}leNwsId71M3nLLCmu)_1|qnOH$j_BVb)7A0t9U zEuDv>C2~_JUe{W+eYBdhgzsQjChbbYb9U6GQ9Z$8qfN}SEje-2b*)eHGTg_4x)LFx z7WI*6*_GDIp{#65L>+>!zdHDRi&+3Y{n}I_hB zi7;VLf8DqQ}0N(Rl33%~2T$ zJw3vYi21voK4!(^7SKE5TKjzBj-x(djB`TUBh+!Prki66*w-yO67%eH!WFqNJSS+_ z#NzqQNskB--7a;<;C2a*KN10!&65mj@B4M{=I369W$6j({@~^ozWBVFffX^&whSB- zg%+=cW-+kFIrc<|sO9Hav<%-=Lk}!>W%8PHmZAd;%e}7DGu?9j)6`*I6cvbhwk2if zMzkzZkHGhI_fsZ9L@h-YXwfTZhWBmXPu|;^v*a9NSah0G&x(|dx5aCYY-?hkZE<}W zik3zeOI=}Ip3fyhL@hbv(NYmzISkfiYOJL(XGuTCurxkOT|b#@9e)|tWmO_E&$e7| z5RR6+PwrKMW&cegMAVW#5iM?8?`nai_PAE#Im_*n42wtjT|AuC+8&4RNEtbum}grS zW`v++Sx;^Jy<&we5h7~2JsB<9*)`O`vhMoJIh-Z&EW4{$ zh^wlnf9!ApeeIox)Yw^c*|viu|VbnC3t8qm|ViioIx>ghFaCLV*H-r^e(Dc;ka zYIH^B#$A;sVHeLUCxW{VNdTrlzOT>zVg-r_`0dKBm%1F={L^}#-6@Xx19s@^fPUV zr0Vu`!3V6J9=Zj3`dTd__Rl^2`jTnb)BCh^cZ8mPPmc(!R!<+Fr+_=gJ&Qd-aO~UJ zlgJc3_U*UCV_)93gh|lTeFqRRf7jESt{dJC&TluA4@&mcsQ({F1@D-Y`I>?x_mFga zAN2f|#60_)Q1+PWP_AnJ*I>IOY7il!+a-$HE|z6Bs$hBiX2c8boDh1MVY#46-M3xT zRt5i2?U0YeJlith#4S9(+1_!&XHFV>5c-+_*{2Y&Qf}tVac?i zzAs4wZ*~I9hC*VVZ3$>V-RpES|9BZJO-qRoQOlCqXtCC?+Xt2zFS_Y+mV#7<#i|vx zF77rS_#5(0dBi;1;xu+2uFH2>8DFq8DSTS@)C zaH>g@!>}&rdK2?(OP14av}~`j4zHgyO^6Uti`87TxGg$57%b!4M33hzY3U40_b1f7 z6R*8vaUavvBIel^r0()snzMqqiX=WNeek~0{VcSES_ zT8#k?yM3UGwM{VjiqunwMG$Cd3QFaPhG*?WkgV!7_hfU3dYmTRRFzvf8qOkHzW9g9+gT|7OV2;z2e!8OO`q+__PDi3zYe;2;= zB{PKFN~M9+m02%y3wgNa7?wwb{N*)=xlMQ`TyqqEB*KK(97jht!z+ajS5wpA>-O*~ z5m3c7$He>NFzNig%VXf0!>gP~s_r$1-~$HM$<~8wjxn<5rN@ImUvuo+=!DlCb9Tpl zhHH+AibP7P|>9;55N~@YB4d-J|}b-y$8?L>&~j(gmXgeG9pBDyYzF$ zbHbwCZ@GKoESBNKJlmpf6^)i-7Y=L%%gHDrMAV}1 zf|iWM%?^Skb!L|m&JyvKVR;)v-A_MU8exqV2V$OW(Kn@@gSH5{g#R{>stXY!YSCPT z7Dd-~C&4o5L!n&6uWOE=0*0k~F7-E?{)=nkbI@5|iFvl=em3eOih_7OacyTVkGVkx`}oHXvtb=>YrK#3CX@ z)S|QmElFw{=YeH>cQ-xG;#tD5tkS0Lr(be@gYV(lJb;*IThio@;rFGz@}9k5X)}xn z5w*y8pk<$~=?bun-Pp&3vy^;eSmw8?Ig19(N8up>aM3P8PU$KX}cb=i~{vMo&6$+`IE^8z69d_|_ zQzD4l#RWb6Ma`pF$0m*mOob5nvCI&1D}B}$^A6C{r%xh6{<5cE9O^y^dioMe zB23uRPfu)%?OHxi=@#_#!?THiDtdZ9I~z>W*>Iu@^zdSgoEOVcHsZjP3ot1i-Pc*BT;I^w{UI5|4c+It^O^ zJw3>mi21vop1%HM9-QC82B}`VxE0YxOQtNo~u_k zF5dv##cl!-BD!5NR^fK>NOH#a|H&@C|CqC+u8|e~o#WtV>ilNf!w|1IMz12~*_O`^ z)HTP*oW=O=nw0fKh^XbQCtCW|lzjl}a;|*Kd(LugJ;U;&;6CnZG1~L+-_!Y4$6q?n zw&)tiQL?TE%7(8inEA$ zw&ilcVYKW_Ik*rkPv#OKqLvIVv}}50sREYP5e=0O{JQ2ix`kosDsvszMRP@IG<;v| zj}!B3i%C7|`HFdc{OW>bbSx1fYI(c{Ewjy^9|g;M*~jfUOLzdoGJiVtZ=RItxpV=` zsd~gb+hQ-X7uTgj$Ll&^xzdyf5w)axqa`!R$_6Y7yMuajmcX41OYmUo`e}5+YbCIx z93_NLa!bWGC-3VLT)9wMbtAq z1y>FBLr;IFLxlWgPp>gMZ3gu8j)p{-u%~~w)WLTBW|#eH__{soLj+XO(}&M+!6e^x zwJM;euNX)qRkx=LKA?3}TR-UOb4-cYKlgOk06*;MsSZnTKu9y`|2$mh z=QZ%1=Zz*2nWD$OuIusG7y0T5-cQ%HBx3%qr*}BM8#~mx8}^Rs>3Pmd^Ps0& z_asvO)YFGwpnCfLOWO;eT)!GX#8uVPZ(NSSwa-2%QwyRphg1mOo_;L*5QZGv=8pFn z4$oqSkXz}adme73Jd;KPp{LiLON9JoPk-}%$O7o;NzO!=u&1lc>5lC>a-B^8eBBI| z5CK*6^aU?HF=>`T!EWg3msS!<)$Qqm4;XysIlg1EU@Z~*=bk>#U?=u;v+cuTpr=3f zBLb_{(^WjFtH6MjhdaS>+%b^I6g&2LYHz_~U%IJX2;5I!zKe+YyPh6hQ{g9^6DDVG z-NzkALoYJU37ZB{=eMDY>*K%MvgkcA&ps#QW$eInf<D! zgAaqH=i%-#oW=hN!}7C|dam=VNv0Qk`Ny3m=Gm5VwFtD#oBbgfEcN4w5K)VdKd$l6 z{J`2^QC4|%le2hVXIN_9x{1f2S&DaLz!K1km}gspCtgF#hk@VlIp{eZh!9bW*H*L~ zd}4^tRz)4zoXJ^AZ!#>~22y{kHq;G|1IzIkVxDc$E~T!YZq9A+6xOBRWgvV0p_ zCaRl_fOYZMGUO9yDY(P1IMk({kIgLqK>cnYn3!iVbt&2su^cQN_Y)zamc;>R zIsbG05U}i-R3;nwtEXq(XIPdxQtzEUmC<1ctV@~|G0(P)o=9B_Zs~T_9xTHhh!9bW zb0AvurpxDmrDX1j=A0$%5yO&HLA{$Vucg6buykES%(E@QePeN5@>j*-vmzDCi4alC z-0f(IADXljEXQN*I&+rfrwmK0rb+nSf4{^5-*K&Tg_vhsn#i9(%Y*%=@Haz|DMW~< z#dZf;_FNis6pF*z(^2aAhnA61n}H}Ti3`UymmNKemrcp6jopA8xgJ-tH` zJB2J~&K;HTetN{Qe)-VTeQy&ff9mOGO(~lE^3pv}u0K2^;;QQDmwl+e7YweqHVC4= zJgXv#M4oR}j1AQKih=|5^e(T7sDJ9|`37U~-8Dns6OrOQy=OMn(+^u8#n;=;`9uV9 zySSjI#}0_XePMma2LAA!H!Wp`kXy+qvrlH4Kb zw?){~8#ozZyY_zZvp#&?>NFq%s_5w&#y*&obj+z0^z?^Kh@|TFbioIl-uJmS+)qEF zM#TQPr(16c$DUrt$Lb#R^kfYpuv$IcHl-i#7)m$KeZg~*4v{H(>}$Uhk9}!7zTJbK zUeJ|@`MaJTl0V5E&IvCMns4Neqw$%HbAqBV^>1J-Ra9&M`+DRFVxE0YSa3TI&j}u; z4{E`7X%h;A24YP(z*I4B(~IF^@b~eCC!wWXIq@Z&Y~r5 zt{uJ)dW$&`B5Ii&glnwUD76k)VjT9I;w%ww7#6=rREJtHg!-xy_A?|TiTdiM@vdb zvrw>Pc@ZI^7PAnvIJ^G*4C@l^Ec=YJ_?TOH()M9(3-@#r?suvIjBGw#@cD zjuxHH6CZ#@&WH#Rwe;DImTiIgy}*)XtJ;LKye(x|jN4Jyf^j}w@cQY=d}5w$8P?(u zS{5F$8v>Svi-{0XOSdqznE4+|g5qFzD8-4hT>e>QPk-EodaYozccvmV?3t&DB$1we z&i)i0~j$Ar?<``BE@_9 zo>kp&8IvZN)`qWZmpmef+rUX4Z_D#D(Prv$w z2>HvN-d)Cc7WDL=KZ!75Pj_}7j_vwZ2mitFb&Hd|MStC#7Fj#av~_c_UgY4kV30!H z8p<;5WDPfL!KAK^nVQhkM<^0W)$Qqm4|wx=8?d}sL)Ys@g2%uImA5soKUbn0nZ733y%fEc9Hu)goteynJ(0J z@tKus4wn7XqP#gvs%LGX_jZQ_qU1?^-hrEGg#1JloP?-9=oN^%r~M{$KJ55uzj6LqiCLa)(Iu+-R3%(E>s<6_WK|AqfB*vA$gB|=0k zYWvV4W3===SY|XmbBVJY^vCAp5xL(UTJ&#*K)N%g^#9^>$L16mDt%qS<6jwCK<1B%J3`>_->NwU)WyS|smzV2_dA3C+iMj`U>Se1{U`g^PLPRYJ z2hg(eL_1xu7`Z0@;4D5t49lQE>KWw*BORT<(s(&B&$dL&qbvp1JCwju?oEVQjhwFeqBHP+{LgAyF{(a%;Rh4fJL?DGU+_qa>186j=j#zHUUer0udr= zsh})BvV()51R1qbvF9vtp|z`e*8cxL&%q+_6*jRo}xW>C?5$I3TOHeaaCS(Y^QvWir3=% zGE+^eh$4|zD_`L?$LBFm;^CUZ+>D6&r)!Q;O6ehR&EYkUh!nr(xNc8fb3B{#7XOUD zXDSiI?c#!Kj=`M{;vO~dQt##PJ(+3E3?a9Y`z7jc0}IadKLghse)dGjUtV+QZjE>f z*BsVPM40fJW9KUBjWf}{eeL1vmb;h;sN!z}%SZ0SYmOaOH%#D~W77&Esk+x3f)BW5 zS=*a%&0+3M#Qyo3BW>Xsyylp5_HJXi<}lqz1Xk;sBSP66cMSP)5AePAeFBI~(PN)l zBp$~X-pR6pYYv+bBIfU|IWA;{`NH|_-U#2N+;OyM596HBt&DnxXMxXsyr14=JTcEc zCtQ4T3HxB_im#91oRB-62oc>brPOw@tXFsrEE@(JZsIIC`x%z5W2t+c4KHjN0N>k- zqlkI7#WNxWEmJJZ8-Zo~L?T4gQXGYATt1-@{@XwcPh^L4mh?!5CGjY=F5`S{&A_rf zj+kd#1`kd|%i-F)o`B_KA`v2LDL90djVp)Z^Xc7+{m*cg+ea7{P0i!@wTX@^_65tv za$=rsQPQG1)b4kUE5P!;_EzbK5VhnUM$4q_(Jx^iOZzk^nX@FGU|3=`sOOPX zNNdW7dA3F62GyZjDRsCFmL9Su(hnhO$vQ$=c6?|CmUE7uo^zHnrx}(Kt&_MeVHO)N zz`DF}Bj(wbS-q*hKW(qmWC~bruOvc5Ezgdk<#FAOx50AnS$H95iHc=dB8yL;rT&0T zFTnEoH8Ib&JanM$H_TErt)=d$`_36|Ad-?{89dDth&nzTT{?yY) z^{0Be+?X*npr`wNCE}{;>FxumzYPpdZ#WsEtjnv2B9R;Se8Qeyc=7f&=;>W+>q(dH zpL+Umb(dMt)76!TNb#ON_88UEL+nZm;Om;DOayVexS*#$dr0;4@X1~qAmp_wGlbkq zUiGObow~HUlL9^cg*p-Pmp%P`!t)c*)1P)E!h}6t&SwU;>%{D5AK>d2qelc((bLCk zhGA0vluO&8r|b0~lB(O&1s^a~`{i`#>AC%g*gyC5IVUb)PrupMXCUn04Cepa2fw8_Wf;kn?fYs4Vc$1p zX(bQv^aW!`OzAv5r*XNrFeY43H6AVXqb^~BF(E3RxovV(IovCfTana@$Ar3i?7g^@ zlnuvWtUghbgvg$kyX?FS^GLV~EDpMfvn7_T5ds#=7Qxs&EoY872luUg15z($X?>dQ zukiRi1%1%CISG+vxp@ZLI6P4o|EIIL=JvG`%g;LkmgP=tJzaDDLv*MpeNr!GIXs%Z z7w0}0n3mF>>9JPa`%IFnQ2Gr#Vq+g z%x(1wM!5q^O(zl}%W~-)vUoI3odGNxettM4u_Qeduq^4p+NFw}?tZpb?V^uUsh%C#Q^T?8Ae*FQkBvs#cUSfF~D`4qznSIaBb4-s%!17Ix)QefB z_hHXDhBTXVWv3i+x|4tjd~JIBw0ryG4EQGey>E~`R*fTx@0lSp|_ z@6^H=hw2r}*WfP+tyS%cE-UF0SMc-~%lt4TBU%-oZCX_jhS0M#X!v(LOJ1hBU%}I_ zRv{s!^z_T8-SAuR8+1sRqNl64%tO1*N_E^0x7%)g5}?G>b5i{wj0Tm z>-%h~U&g*q)lv5-c)EID5>q-)H+;73G>mWkf7EUw^`mEA3&sR(f3~JpFu40r7!&rY z)+wqNj|q>?F&`Y2vsf@{ToIB%hGBjM3%)q1X&t9QfUk<;bm`FNh~&*0+yw^jAdS* zE?a@+X(duGX1U&h{o~9iawE>4ChL$8Sr(hC$kKJ0DgNVZOW7rk5=+ik0n6$Q>^m8z z^Xh&8mMN7;y_h8+md$H3U+C9>cG1xxA+jv%Ly<*uM@vg+7nh;+4oEC%-vul;E!n-l z(>dxfucuPr8Zx)-lADVxHKvgeSr)5n$fEuvbp){J--!;ASQ7FCEEeYMn?hsE zqZR;*@ncdiW=ZWDhyB=%_%gV+>g!7qBFnPmI*;Cjx=aAW);>g%WIWyDHd{|0rZxToq?Gd)r%*X_NMN3x{bordczVCnB&9e{_Z!Tn zAV^YpJj*myiaJ*F>2JqlG4MIjR+($GL%YFxAjPv3isME#YguWE2| z0eJe!Xc8&!>CN{sPakiXj=zfB7)yerOI*Rz%NP2hjx~N?1K$&LdMON{XUV1FH#|$7 zVp8nD(<{9tA*J;6*G?-#VLja-jf5$Bx~}JPwCmlORjuH5JMx7DDDm`GvBi6O1)^b{tTY(ruMF=gDH`xo9ZpY3!~f* zlM^r|b=4x7vVC9GFzows77k7TPp_*@VoK-f`&zp?!k94W-&{SZAC37X7!$g$Vrw_K z*WGYW_f+*CMfKt_p~foq>|1lC9llHERE31do|pOTyu3DZe+ewMR-Q%@%dIWt75Cal z8ngWs)(1Ki!uWQ76R8)oObt7S<6H8)$GG42tUU>lWtka{ZS0!g&KOwc%^1*MV!5(S zz*2iB^K`e)Wt##^%LSxf%rY&RwTnrPsUcji6;_ZCS(a%xkY%KYO%$*k9+EpzVhP+X zU>QIB4Bj?2bG3T|OZTy)Ud*y6hOOH^|N5a4uymV7LS$Jc+(ecYWxYND%dLyY7fCEW zy96v#0@-uve$ka*0n7C;QZHs1Kbf^lp}&R=uq?kvLS$J+MIcMY$e!1M<=+F7HcKq- zt^$?=iEMwxgC1KZ0?U$Pq+ZODbepX~SFNjA23Up#kq}vyp|_AFvtNTPz*4S2W4FZO z=pkUyJH@{FvGz)ncfj&MO|z(8%rbQ_YnO>5yeb1ryNV=4mZkq~WNA69ns^Y#EvmVXo zf~SA!N>YmR^wiDF(_eIM9|Lkdzb}bX=IK@&n5W;}?raWGUPF~cQOJNEY)8kvcGsqW zr#~D=qW;R$M~7Yg4xX+yokYredYgsJ)6MOl;5)I^=a3-j5?Anahf~bcFLi8}4fjc( z#ljGJmR8SYXX$dJRd4Y0ldDKbDLvhxLP&k^bVF+rrs(M^J{!=k>*(6yr=q4ikN_o~ zzBvCpCUvZ;cPPd|5%1eVCt z=dWLnorK%3hWM|Q?*1fGuJ5zaip0LJm-)C=;ORSpNKENGeQ=kn6<|y#|KhOfsXyx- zS^xE;3h%_uJ;%Hw$Z!w-gXUTisTYq4{V%h9FM~%{Tma)+y^kbB?!2gUVdrIaHGf=( z@^9Furo@u$BVd`ila1AHA1p)f@MuNq#Vi{;v+-^8vd8OzWt1@qk!5Lr7u&dFOht2O zmy63jHIrE4`~)lxtJ(bNV`1%!&T16pCXE(K> zz!ElOUk{1p{&4||({Q#1ef`LJ>`#5hka{u85l6P3KJeP_Y+$jRNvG$GPCb3NuoHrS}99BFoYs8d>H> zog4%#<{kyJB$f+j1S|_{v$>#cn@zayW#&OrFJ|$&9D*!%#(H#EUU+B^KvUrJjC!GV}D^S6zC5VF$k>Niv>ZC57F8Z4OVa4JoPV;uI=p zduB6F4@{|e3_LyI8%Zh7(=Y8}o?h?s+@~Pd9|}mEGEcAGk9oT8+Wokm-mC1`qVvt8 zC}bZ8W}velzNrYFZeEE*{gtQ3?FbkHp5C%1iIn&B-4~gs$EU8@1edr$JrX2c;tHOw zQJ#5v$AvlrAw<2gFod3^BMaE>pn>}?Z3j>9*qVft($n7W{GRbF*xYo}Mv>#FWm{V`jYb zhB2X48<+px*Dn9{qY9qhr9WF!Yp65sI9%&Kok_iTOlYx*jc>Z+zFEV0(d$D(FTe2Rqoh?URdvQNWUwN9x5a-OsY|?eVlbEr8`gndU_Y zA|pREQO54FTo4v9Jiyk=t(S}BLytlri>+IR^2hcay6FJi&-}BX5UEBdfR^vuxKWb z5LuSo$Jj0>B8PN>Yi!dLBO{3=DO$i%UY~s*Y0~up_@9uJ*`!{~;_Jo!d7BZo6Zdp) zUQ9w{S+Zi0rB>H_p1`uF@0)%S%hN{!mT4iZU8=urUK!e@(r8jIW@)0&oZHWiz|!3*!&+iF6E9%-M}_St|FX&qzm2xVgVc*zmbkKYpc;E` z;Qu*dyh(^GOWbp0nbkKh6eLKeZN?3W#UVlIoWr^vdkxZeb^a-sEd+NZNiuVe40ra8 zM8k|M4UoJvzt>jd^WId=XO1qa|{?vQi_{%G`-B`9FKjgwT3x|-AEG0CkmEJ zSFZg3&m2bQvGKUc1~oT`nl?d66oow7ne8Iq@~G}tm~(71BT;`f=hz*8s13|HjOLO^ z`8mh=*KE$wTF>_wbfcRWlOX95SD14|-aUe9Sv9g3de~wsVF*1-qt>uJJk3nZcEX$^ zd;eCtOkrL9B~#&cbJ#%wl*~B>)VYaCKO1X3fjLJv zSCUlHIfudlwwtneEX+A-d6L-T=NwBOrsAAq(2wU`Vb0Oz5D6^NoFn_Y16~-~k%QVp zKR(KzWXkn@Hfk@h@5?S6tPjry4xJ`3rJHlqU($0bj0x}DM#oD1sNWmGm~bJ0Jue=0 zC>Q@p*vgyKi^qhv*V%LYUSIVa!+Fs^MndGyi%KjzFHQGN&IA^>3i&A#i|2a*%dfBO ze-tC8Y{makgj6H-ViwiT%saB&bIgEcbsZ8S%W^*s+t?^p55Jdbu;xUb#NzZpz;bm9 zYnK*H&9tE-`?{Ibi&-AKF;9P`a}M7P@pL31vMhIABFoKxe^!BZnYwOjrJz6aC!0?K zmIi0op6*4h?eY6a9=4=j%yKVHb=kwYbNfknpBsB$g*5WK$Z=AZSfuQ7_U`>C6)IjlK6nyEN%a>cuRNS~5>>w9O0OO}T7BLS$KvzeX0VpO!yC9G-+O^pjYA z6)5#|+Z^_6;KFqyA294ekt9jR)2AL`eSE-ywr?Qi;6rf=m9uT#*t3D)F=yL>r)xeZ zDaCnuc@O647pq1b1G#RQNaB=v`sAAIo^)1!z67EQ-z$ltkb4K^;x6)wrw&A}* zQGey>er_*1fTu@flSp|_AMl=e`g7wbBe=vj=aC@k5?Ao_I^5H(cfQ;WA=WC3imuI~ z5PFt&k7dsWEbXj^fTvrjlaNw+dilVJRp9AsE0ZuqPalxrgmzsnr2xM%xlo4$DDm_T zZK5z~{NB4e!PCF#k))D(y21fg8><%to*v$a#1`-AD}H98r@Q|8fZv#0+JXd@$kS)c z+J&CpJ?)z}^y78flT5k3&*op&_h}mpj{{F{(UruM&eNBkb-{m!_l#Y7R_aG@{VNy~ z>Q`lZMFxJWj_XhfuSmUkOt|XIzRfl}C*&5Km(}k{h}?NmIr0X_gqvnz&A~hD+UeYu zSgzQrD?aB?E64VV^jJR;pXp8NM(V{Z4yt!?eeh@gJ|AGI(vO75vK&Z47Om|@V}NDg zo*QuzOQ5}gB_^M(K?mnNnGAi?wj}jpmd!`mKh89#Y0d@~{f;C=mc`{AvII}cuRX-`O6X`aj0j0hS+XjfxIJmSx9#WHGqE zE)cG!R?XES5_3H-OJ`Fc_>W?2x&?w6-358gn#l(8crvMkmq z$Z~6yT{^I6zUbLqV#)Fnu=L)>)>N~79^nRuYDuJC%;IvEU1JOPM&o~#`=*f)S(de_ z$kK3rnSAi{-}lb%l2~%~t1H`UKQ@G|r+>JX7Y&9z?-@yw@pS#w?7M;u`y6c#DMu5; zDOAojglOY>dSKJlTfx&myeBEedHSA1%+r@&UNQpYy7?y(r_9rH-?O%Ns<E&Zu65E4^G7(&m|jAqQ!%S>C=3_QJ#E(s~6r=K&>z`ge0>X9%-Pq$5S zMZ5ktr2iSX-4Yv<041KjZfy)EY1yaM1y46>MUqPD=?VvEGk9Pn@bsn~NNn+*e!1WW zdV2lM%}&62dW&u(utc7&xCL~j~@5^U>-{C2b48hYihme@kdAe5D zq4Qx(NNhBAuhfqwdke;dm%(h^_RYTb_$|YXR-|4$CfrSD-~Wp-xpxQ7OK)QmB70s| ze84f`(5Z)&fyE-JAV6Y?J1k(SR+;Tda2?a(1l+elA4$EKWt2Pfj&Iq4TEMa~n}o=+ zSf(LMUH|>~EyMj~PhOW;?jIGf=v$t`OYNYe^IKpEs6^_;ED`tF8nl0dviJ_Sj}8fu zWtsaCS$h0>TMpV~$!5gEv0;%t-wEJ@x51`>-)uz;oQGB$r2vh&&DD5wYSFDrcLWnWvjRu5AdOe)=UzDbCa9^3P|E-giS}H~jXr5v->- zszJgOJ^e)T9<*zdgF8>c?PjV+0+e|AP2U%obZV`sH+Xt#Ly}ZdPggj=b)9~U22U^F zg2Wc@={HSO@Ho{vy&VXio@7JHU)MenX-M~ch>h==QJ4wo<6V- zi7B0@FKl|h9gGQa)h4f#`q8b|1Y<&8N45^tOLKN5xYnQ7A@$-h!M-8eD^j-d=NmA- z=`|uDvgf7m7aSAD*_+}&Z_kGRa*|knh6`B2Z?bjU+=P&iz%uAQsTZ?2-C`^SZ3lk? zmLbnch%8HwEM(c)r(r#4ml;t3J`&64TLPA?32fbV>h|nbz!DZq>cuQw>au@n9o&(Q zd%8!*lMq>!PG6B_dH;j>jY*>`(=SLYNp}S-@hzC!TF0t)2bQKcNWGZlOAK32ZRnb(T#Nl0u^ z`v_cPs`E*REK7@T$Wq6-68={?aKepviREsLfW@#A+h4J~#}oW7Sc4g)Ud)nxn|XTN zqv;yJ5;>oQ$g&vbAj_-P$MIY6OMb5SBC&)%6|hXd!u+Pg);fKmT`s7uEUFi?^fhMl zr%d0tGr+P%lZ42!=zm9+N#^Fb?( z*fsXMSEZJ~V%?R5$gxNkF$-U>b*`l(ib4jzWuD%4=;Gbr>603esK4^`d#5X8fv3-EN+RVwU12>v zKXW-M_S`lkNV>!oJpDHJ^mR4|@W)TPjfElfEa^BiPY<~D#u7YzeoqooN>5L0;c^)~ zeOZ4Jrs(PC()OdZW?l;z1-IK8Qxc%W)2rWjjY&t`eRRRo6 zOeV3#dwSM_vUr?vCxb%3({(LKV2M0^Xs9=Odisoz^3eBvpHDJn`@SsJ_f>3~I~F|s z=TZ_=I!`aVc||CA$HOkZlcj#tFF`OST$sYVBRz5JIT#bdO-Q|XOt7%<#r44s&+LNW zyc`}vLS)a2_75Br7Uyoj|0}jG-)phN;+Z61IlYYaaTX!&FMwru6;dx|Sy0^%S=!u* zUIi>KbV-OTOU0kaQa-qHD`=PIo|!fhi&L_I<-l>)E|-Va@&XneUs5k-QLDtB4GfwV zz8zR91dmuJFs}YIP5O5*rW+qBA&4Q>0X(d_+9(>BvLPCG496Rfe5Il zi$3`HBMFgZQOQRZ7mo?9z;f=|nBx*l&L;s&-y`h#Q*ZrewV_@9xl8KBEW5nfbB^g6 z9dQ2C?lB3GWyxnO?E6UP z!`dbS%ewm{M3yD95Lp~`)O!F+hx}#lC6<^U0v5ZKjOCE&3tUg{TW)z#y_jW16?To8 zKG(o^AY3bv5LuSAU&s=bHyg*7#D|T4NG!Js1T01#?B3t4Tkj;aOY{p;FJ{@|!|r`O zy8!IRJd;R>EX%upktNS@v=>N_!|gWR&i|P|J^PR4e-gC9V!@KO7PDrpT46D3-T%Ms zSuvH(IV`rEh=AEb;voTJ7E_H5wDv9z&piCdO!TXb1Tm$<^5qv1LBY#{b|?LH7P zMoSn%&(fZ&>}y4B+7E34a}Fm=JhhN7}i($@DSet|?&N;$9`=Ei>?tJV!+-{o; zNPv<#$C!i^OiFE=KM>{|>di<}N#`622WY)^YaBcq=+lMJ3@kjtLsR ziTS`{u)2(e#1d#%L2<9W$1k?lS#7k+FSu`eZYK3&mZ2u>8!0;ad3nI%=|DncSr(}b z9;&6Op{c%dmW8Foij{W;?n?lc6T0WDBo-eB0ZWa??3vAg?e(pI#kT>e7qg5T$9(Wb zchw%ia=tkUk!49^EP=KUh5<`A!?}(Ui~CLii^gL1y@^e>CT_q|zcr~Bv-qa7e*>w% zo$vuzEIN@8S(Z7f*e<4-EAbujGFxgNkXRg@1uU)Kv2RSCT(~M3Si0{Z^0k zt@LT_0$@4hMnYs+-Z7S{-OO)7yDaM)6C|+|x(Qe&d}l26Ue(0+u1^gk^8K?Z;<*08c;tltlfNr!NSu@(MitS3HT7_jJqq%+sIO zI_D3U`1!oJl*#(^Yk5kZa0Gvt8`%qJxiv(Y_I*#PQxmJr<-MykWzZOhH2SM z@btlXBuvrMKV}_8Yt8zchg;?rsJ`d7+fpkFOY_Y`7^`(>Uur4!qj3iXW5Rt0_CJr&Cxe~f zTJOG?)QiW2S_jyepsCVR7tTxiDiR`hUR0j5^KxeVBit);?6FyIiRJzg0n4W4Y~Ra- zFWvFIfzPithj6T?)ljRm*$z4xEWLaVui~2>k>cFza{nlKGg2B-jT002X77HR~moyi*zmV1n4O~khLz%n+v?LQLBfpY?u-J#3} zXTE%e<6Pu3QZHt4xyA063U!`YLc2J>CLyvcgUVyOM5ZlQ0hZglG7d{DE|&x>nlst6 zwBL0d@q2S_dq};QC9MftPj_sy_bae$JxD@iS#B{F`{c?;faP_$Qx_$ctsw#ySIaB7 zX1qtYGX7UNp)#o#vuvw!39qpLuQNCo^wcFGvMeU**e(sn*WL`Cu5NKHM`8)Rs?^h; zjA3dP5acxm414?{k|g8la`AT4^bB#l|)gij4LtpZ3yJzGPoEYr zsUvv$<-H_Q-qV`~u#2JT`*>p*5*i&MLDD6z;OVzJvSm~K9zF2cW~F1o5PFtwjAeUx zo=?@&15dwkiiDKX(+7F(7z&<#@B#@_^mK#o0cfp8(+Wny?G_PA0+g($-#+sVle~SN z35z>gfsxsI{=$SMc;{_egB;X5-DMwN-9upR?VLmuNNFV=O1T4=kv-hQb zbuo+t7SF?^Ud%FZC0kQF)%g@Y$9E1OA+jvpE8zGRrrs9+3ugHAelv+B>9&C7RXlqy zZfs#RzTDpu6$?DIXbx7;8;B)m4wK$TwyF1EIvPlcBvAwtB1t$G)lmt^MgId zcZfbY589a zZO@qpCPKRm-B0SpEM|%98(cx*_a_2NvM&jdWeH>~A*X#C0n7MdJ9bJeelGsBWv3L{UhqH*DWaPTHg(@N_L}67^S}?w6La z8$5mTHWDfC=~hn6(;L^CoC24)g%b&qE^!4FFu?8cyKpE=Ne1qNmTu3qor(yBoU@Zny3MBtVI$cRHMpNjFV2?7-8b z&XA;%db+{^8qV%L9XvhYGKnpLr@yI=$C=yMy%~7=rRyZHM4o<2gDu*0AN3l)dwlT@ z$&~H;7P7w2Stqg@c>2i)B&Kwpo*ov1dxy`*&QFl~QO`udm@sAo8>`J6H@AapJwBe) zi^qi7QOr9kjy{Rs96yvyLS)a&F?L?6_^<2@CgPhCPl!~o9`X&nt!-D zunZ3;^^6{FxjLm zzGrjiHK`Y~yfS3}pPCSOQwQ3m-UkvQ%c5Hu+r{@_EIu39F*$II#1fMuV3~iM^ovz8zZsrQ{ONGzTONBe))Wq_yKr;?Q7JpDp%=II+c zcW(=FeI$d#Df9H6w^`dK4Bk>7q8xLSL{Z2TRc4_6>&F>_r|&EzQGey>LG_k>0Z%`p zwyWrBkoRZTNj+WR02}X%H2_cV*O|l?@9Bmgb&6c-w)_PXqs??8ODHMzeGdblc^O#kJFw zX~0sUd~i{{n8oQi`yTY&?N<0s?5xTpM3zOXDvodQWt-tQCc~^3zLQvdwhCBw^=A$> z;AIc|M#=#PQZHt?pUNDn_3fTrpk02rkPulGN5*nXJHHpS%Np;7xe|-Jqk!eZ+dKHI zIxae8612ZnB^Zm){kZMd^8AHhE^C}bP%#E8@2Ho+g|?~zBgd6r!_!gN!}}9 zQ9aJ?{ffW$;Qt2JHYW9A7PWuaoa5pdn-RcLrVR;^WyxbKZKqZ9197nGo96Mq`(?j| zvi<2hMzHi*Cpwifi7Adg_ynG=vXVso zm8V}H?xqKxp1Y1j%6s}YbLQ#G&c9y`w`=t+BuKi%6+AuaAln1AE$XyBgiLo7hS0O5 zp70g_Y;`T&VmEmD6=xDsN>7hpThR_Yefu5~rs(Nzsv&5tSKsWJ4!7IPgCszSr&pO? z7Cqg|EUX=PdWauMDygR{9N?7MA76u~Tc0Gc#d~_S&vlXb{*qbOz|;RZM*>Ua=}R)M zqi|f=bpXH7QYD0B%JzLTt7AWYy3YP|@bu9)NKENG{luAmS}?xFRqgF1^`mkB^`i!w z^)<`V_vdx3*du~=bDr*lYdxzrsTYq4t!}V)FWZ(m_#MWC_63GUoxyJ%17+%kqt}%v{}P8?YFC z^$M3*!hHoS;mK@WW%=EV&A_rEpVW(4%nKOHfp-0(fn}uHzM_MWWm#4Ouh*RgRWyLb zbJ57B63YdD0n7Q>H}JNpa>vdBSUy>idNIq+X!guz*z-sD{lBN1NQf*;24gvWy3-b5 zv9m9iCb1kpDPY-^#=dKxomU<`{kSu!7qhJV%=U?0sc;tm4fK0236W)4SQFbN>*%JX zz*1@4wE~IdK#+i?N^7=PWaq5;TcBOcr;~ayOGxcIIDg8#;%yDr*wZ;AM3&_PW7#w0 zmOijN%wJjM%Afg@%XtCI!9uo#V%o9}{#SXOnpIJ~nB|-0ZDi?uvXeTn7*-@9vMjT8 zuw70r?ehUx$_!}JNMhM~S->*q276``dZ>9>Xcz4fq+ZN2PK&pTU!o(hyqZ8lWLe%a z7SEgq_zmN$pFebxSbl~GSayf7xnR35X$HV@J(tvrS*%?muw5bxlkoo>EmYHr4nmg2 zOc&e5UNtozEGn_z$N$CT{l}tcQTxkvn!AIAoxYtLZ(Up|h+ Nly3aD@>#q4{{Ta7*6;uT literal 0 HcmV?d00001 diff --git a/logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/trial_1/flags_0.json b/logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/trial_1/flags_0.json new file mode 100644 index 00000000..c86bf341 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/trial_1/flags_0.json @@ -0,0 +1,63 @@ +{ + "logtostderr": false, + "alsologtostderr": false, + "log_dir": "", + "v": 0, + "verbosity": 0, + "logger_levels": {}, + "stderrthreshold": "fatal", + "showprefixforinfo": true, + "run_with_pdb": false, + "pdb_post_mortem": false, + "pdb": false, + "run_with_profiling": false, + "profile_file": null, + "use_cprofile_for_profiling": true, + "only_check_args": false, + "op_conversion_fallback_to_while_loop": true, + "delta_threshold": 0.5, + "tt_check_filter": false, + "tt_single_core_summaries": false, + "runtime_oom_exit": true, + "hbm_oom_exit": true, + "test_srcdir": "", + "test_tmpdir": "/tmp/absl_testing", + "test_random_seed": 301, + "test_randomize_ordering_seed": "", + "xml_output_file": "", + "experimental_orbax_use_distributed_process_id": false, + "experimental_orbax_use_distributed_barrier": false, + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "workload": "wmt", + "tuning_ruleset": "self", + "tuning_search_space": null, + "num_tuning_trials": 1, + "data_dir": "/data/wmt", + "imagenet_v2_data_dir": null, + "librispeech_tokenizer_vocab_path": "", + "framework": "pytorch", + "torch_compile": true, + "experiment_dir": "/experiment_runs", + "experiment_name": "submissions_a100/ademamix_golden/study_1", + "save_checkpoints": false, + "save_intermediate_checkpoints": true, + "resume_last_run": null, + "append_timestamp": false, + "use_wandb": false, + "profile": false, + "max_global_steps": null, + "overwrite": true, + "hparam_start_index": null, + "hparam_end_index": null, + "rng_seed": 45840584, + "set_pytorch_max_split_size": false, + "pytorch_eval_num_workers": 0, + "capture_jax_trace": false, + "skip_evals": false, + "?": false, + "help": false, + "helpshort": false, + "helpfull": false, + "helpxml": false, + "tfds_debug_list_dir": false +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/trial_1/measurements.csv b/logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/trial_1/measurements.csv new file mode 100644 index 00000000..b7475850 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/trial_1/measurements.csv @@ -0,0 +1,451 @@ +global_step,grad_norm,loss,train/accuracy,train/loss,train/bleu,validation/accuracy,validation/loss,validation/bleu,validation/num_examples,test/accuracy,test/loss,test/bleu,test/num_examples,score,total_duration,accumulated_submission_time,accumulated_eval_time,accumulated_logging_time,preemption_count +0,0.4999999701976776,11.58028793334961,,,,,,,,,,,,,,,,, +1,,,0.0005500487022288,11.740890534578584,0.0,0.0004835649898947,11.741181138485574,0.0,3000.0,0.0007088489919237,11.744577595723666,0.0,3003.0,53.060325622558594,499.1922066211701,53.060325622558594,445.425760269165,0.0,0.0 +1,0.4999999403953552,11.56773281097412,,,,,,,,,,,,,,,,, +2,0.4999999403953552,11.577584266662598,,,,,,,,,,,,,,,,, +3,0.4999999105930328,11.567296981811523,,,,,,,,,,,,,,,,, +4,0.4999999105930328,11.54425811767578,,,,,,,,,,,,,,,,, +5,0.4999998807907104,11.532217979431152,,,,,,,,,,,,,,,,, +6,0.4999999105930328,11.506816864013672,,,,,,,,,,,,,,,,, +7,0.4999999105930328,11.471416473388672,,,,,,,,,,,,,,,,, +8,0.4999999403953552,11.432726860046388,,,,,,,,,,,,,,,,, +9,0.4999999105930328,11.402920722961426,,,,,,,,,,,,,,,,, +10,0.4999998807907104,11.35958766937256,,,,,,,,,,,,,,,,, +11,0.4999998807907104,11.304051399230955,,,,,,,,,,,,,,,,, +12,0.4999999105930328,11.254443168640137,,,,,,,,,,,,,,,,, +13,0.4999998807907104,11.204854011535645,,,,,,,,,,,,,,,,, +14,0.4999998807907104,11.137566566467283,,,,,,,,,,,,,,,,, +15,0.4999999105930328,11.076220512390137,,,,,,,,,,,,,,,,, +16,0.4999998807907104,11.031241416931152,,,,,,,,,,,,,,,,, +17,0.4999998807907104,10.959343910217283,,,,,,,,,,,,,,,,, +18,0.4999998807907104,10.895500183105469,,,,,,,,,,,,,,,,, +19,0.4999998807907104,10.841681480407717,,,,,,,,,,,,,,,,, +20,0.4999998807907104,10.763345718383787,,,,,,,,,,,,,,,,, +21,0.4999998807907104,10.691622734069824,,,,,,,,,,,,,,,,, +22,0.4999998211860657,10.628252983093262,,,,,,,,,,,,,,,,, +23,0.4999998211860657,10.552657127380373,,,,,,,,,,,,,,,,, +24,0.499999850988388,10.478059768676758,,,,,,,,,,,,,,,,, +25,0.499999850988388,10.40339469909668,,,,,,,,,,,,,,,,, +26,0.4999998807907104,10.344446182250977,,,,,,,,,,,,,,,,, +27,0.499999850988388,10.272173881530762,,,,,,,,,,,,,,,,, +28,0.499999850988388,10.20293426513672,,,,,,,,,,,,,,,,, +29,0.4999997913837433,10.140222549438477,,,,,,,,,,,,,,,,, +30,0.4999997615814209,10.043774604797363,,,,,,,,,,,,,,,,, +31,0.4999997913837433,9.995413780212402,,,,,,,,,,,,,,,,, +32,0.4999997615814209,9.92104434967041,,,,,,,,,,,,,,,,, +33,0.4999997317790985,9.868748664855955,,,,,,,,,,,,,,,,, +34,0.4999997615814209,9.806760787963867,,,,,,,,,,,,,,,,, +35,0.4999997913837433,9.743227005004885,,,,,,,,,,,,,,,,, +36,0.4999997019767761,9.700457572937012,,,,,,,,,,,,,,,,, +37,0.4999997019767761,9.642682075500488,,,,,,,,,,,,,,,,, +38,0.4999997019767761,9.581657409667969,,,,,,,,,,,,,,,,, +39,0.4999996721744537,9.510013580322266,,,,,,,,,,,,,,,,, +40,0.4999996423721313,9.489005088806152,,,,,,,,,,,,,,,,, +41,0.4999996423721313,9.417152404785156,,,,,,,,,,,,,,,,, +42,0.4999996125698089,9.377276420593262,,,,,,,,,,,,,,,,, +43,0.4999995827674866,9.300387382507324,,,,,,,,,,,,,,,,, +44,0.4999995231628418,9.285270690917969,,,,,,,,,,,,,,,,, +45,0.4999995529651642,9.213207244873049,,,,,,,,,,,,,,,,, +46,0.4999994933605194,9.181389808654783,,,,,,,,,,,,,,,,, +47,0.4999994337558746,9.16052532196045,,,,,,,,,,,,,,,,, +48,0.4999994039535522,9.116455078125,,,,,,,,,,,,,,,,, +49,0.4999993741512298,9.048582077026367,,,,,,,,,,,,,,,,, +50,0.4999993443489074,9.039510726928713,,,,,,,,,,,,,,,,, +51,0.4999993443489074,8.984659194946289,,,,,,,,,,,,,,,,, +52,0.4999992847442627,8.952940940856934,,,,,,,,,,,,,,,,, +53,0.4999991953372955,8.937579154968262,,,,,,,,,,,,,,,,, +54,0.4999991953372955,8.916813850402832,,,,,,,,,,,,,,,,, +55,0.4999991357326507,8.902471542358398,,,,,,,,,,,,,,,,, +56,0.499999076128006,8.892631530761719,,,,,,,,,,,,,,,,, +57,0.4999990463256836,8.829031944274902,,,,,,,,,,,,,,,,, +58,0.4840249121189117,8.805424690246582,,,,,,,,,,,,,,,,, +59,0.4715396761894226,8.788590431213379,,,,,,,,,,,,,,,,, +60,0.447908878326416,8.782703399658203,,,,,,,,,,,,,,,,, +61,0.426025241613388,8.731322288513184,,,,,,,,,,,,,,,,, +62,0.3973809480667114,8.742698669433594,,,,,,,,,,,,,,,,, +63,0.3877594470977783,8.694246292114258,,,,,,,,,,,,,,,,, +64,0.3582267165184021,8.717243194580078,,,,,,,,,,,,,,,,, +65,0.3369593322277069,8.674422264099121,,,,,,,,,,,,,,,,, +66,0.3312018811702728,8.658280372619629,,,,,,,,,,,,,,,,, +67,0.3243274092674255,8.61693000793457,,,,,,,,,,,,,,,,, +68,0.3022405803203583,8.624625205993652,,,,,,,,,,,,,,,,, +69,0.2901933491230011,8.617016792297363,,,,,,,,,,,,,,,,, +70,0.2748120427131653,8.607769966125488,,,,,,,,,,,,,,,,, +71,0.2849236726760864,8.58910083770752,,,,,,,,,,,,,,,,, +72,0.2735233902931213,8.570942878723145,,,,,,,,,,,,,,,,, +73,0.2547514140605926,8.567353248596191,,,,,,,,,,,,,,,,, +74,0.2555278241634369,8.560566902160645,,,,,,,,,,,,,,,,, +75,0.2436864972114563,8.565115928649902,,,,,,,,,,,,,,,,, +76,0.2457752823829651,8.522514343261719,,,,,,,,,,,,,,,,, +77,0.2341547459363937,8.518010139465332,,,,,,,,,,,,,,,,, +78,0.2302069365978241,8.536039352416992,,,,,,,,,,,,,,,,, +79,0.2383005023002624,8.54018783569336,,,,,,,,,,,,,,,,, +80,0.2278693616390228,8.51799488067627,,,,,,,,,,,,,,,,, +81,0.2307108491659164,8.475534439086914,,,,,,,,,,,,,,,,, +82,0.2239596843719482,8.484325408935547,,,,,,,,,,,,,,,,, +83,0.2169794589281082,8.51485538482666,,,,,,,,,,,,,,,,, +84,0.2165802866220474,8.4898042678833,,,,,,,,,,,,,,,,, +85,0.2166336327791214,8.464410781860352,,,,,,,,,,,,,,,,, +86,0.2144936919212341,8.46013355255127,,,,,,,,,,,,,,,,, +87,0.2182470560073852,8.423998832702637,,,,,,,,,,,,,,,,, +88,0.2188539206981659,8.422173500061035,,,,,,,,,,,,,,,,, +89,0.218675747513771,8.471739768981934,,,,,,,,,,,,,,,,, +90,0.2117759734392166,8.431052207946777,,,,,,,,,,,,,,,,, +91,0.2139405012130737,8.445631980895996,,,,,,,,,,,,,,,,, +92,0.2239573150873184,8.447044372558594,,,,,,,,,,,,,,,,, +93,0.2144366353750229,8.410533905029297,,,,,,,,,,,,,,,,, +94,0.2203715741634369,8.415111541748047,,,,,,,,,,,,,,,,, +95,0.2183998376131057,8.394234657287598,,,,,,,,,,,,,,,,, +96,0.2119191139936447,8.359619140625,,,,,,,,,,,,,,,,, +97,0.2084416598081588,8.40184211730957,,,,,,,,,,,,,,,,, +98,0.2095620483160019,8.386054039001465,,,,,,,,,,,,,,,,, +99,0.2045255154371261,8.402280807495117,,,,,,,,,,,,,,,,, +100,0.2154810726642608,8.361786842346191,,,,,,,,,,,,,,,,, +500,0.4999991059303283,6.754341125488281,,,,,,,,,,,,,,,,, +1000,0.4999992251396179,5.510748863220215,,,,,,,,,,,,,,,,, +1500,0.4999990463256836,4.603616714477539,,,,,,,,,,,,,,,,, +2000,0.4999991357326507,3.881467819213867,,,,,,,,,,,,,,,,, +2500,0.499999076128006,3.4664809703826904,,,,,,,,,,,,,,,,, +3000,0.4999990463256836,3.1203725337982178,,,,,,,,,,,,,,,,, +3500,0.4115718901157379,2.886350631713867,,,,,,,,,,,,,,,,, +4000,0.3970949351787567,2.794280529022217,,,,,,,,,,,,,,,,, +4056,,,0.5461362597165066,2.615983832018747,24.72222418466016,0.5493918240319401,2.567170586849512,20.598851036749902,3000.0,0.5522630875602812,2.5806624397187843,19.29351834035134,3003.0,694.9455635547638,1346.2628633975985,694.9455635547638,647.8665144443512,0.0338814258575439,0.0 +4500,0.3634401559829712,2.6454765796661377,,,,,,,,,,,,,,,,, +5000,0.3161998987197876,2.51291823387146,,,,,,,,,,,,,,,,, +5500,0.3196495771408081,2.416013479232788,,,,,,,,,,,,,,,,, +6000,0.30279740691185,2.4035415649414062,,,,,,,,,,,,,,,,, +6500,0.308595597743988,2.353584051132202,,,,,,,,,,,,,,,,, +7000,0.2825440168380737,2.3867745399475098,,,,,,,,,,,,,,,,, +7500,0.2745710611343384,2.295407056808472,,,,,,,,,,,,,,,,, +8000,0.2545797526836395,2.3342952728271484,,,,,,,,,,,,,,,,, +8124,,,0.5977578372947919,2.1133938968422497,28.960536769195457,0.6121684790021202,2.0209432462089745,24.8842243333632,3000.0,0.6148044855034571,1.9902086601592008,23.51979872328674,3003.0,1336.9580190181732,2191.3487401008606,1336.9580190181732,848.3313324451447,0.0684435367584228,0.0 +8500,0.2408380508422851,2.1986334323883057,,,,,,,,,,,,,,,,, +9000,0.2398717254400253,2.141063928604126,,,,,,,,,,,,,,,,, +9500,0.2534885108470917,2.1668410301208496,,,,,,,,,,,,,,,,, +10000,0.2219618111848831,2.075757026672364,,,,,,,,,,,,,,,,, +10500,0.2135020047426223,2.0444116592407227,,,,,,,,,,,,,,,,, +11000,0.2206866294145584,2.109749794006348,,,,,,,,,,,,,,,,, +11500,0.2091340273618698,2.059623956680298,,,,,,,,,,,,,,,,, +12000,0.2062177062034607,2.179377794265747,,,,,,,,,,,,,,,,, +12192,,,0.6143790849673203,1.976550205612415,29.92785405711678,0.6319698453831942,1.8417314648919416,26.584652851785467,3000.0,0.6400325373307768,1.783123801638487,25.529453286545973,3003.0,1978.963494062424,3032.022760629654,1978.963494062424,1044.3916444778442,0.10451340675354,0.0 +12500,0.2023797482252121,2.100863456726074,,,,,,,,,,,,,,,,, +13000,0.2029584050178527,2.107184648513794,,,,,,,,,,,,,,,,, +13500,0.2025795727968216,2.058523654937744,,,,,,,,,,,,,,,,, +14000,0.1933506280183792,2.0877232551574707,,,,,,,,,,,,,,,,, +14500,0.1849861145019531,2.0766091346740723,,,,,,,,,,,,,,,,, +15000,0.1867569535970688,2.0195677280426025,,,,,,,,,,,,,,,,, +15500,0.1791958212852478,1.9581609964370728,,,,,,,,,,,,,,,,, +16000,0.1737920045852661,2.037226438522339,,,,,,,,,,,,,,,,, +16265,,,0.6276181755751402,1.8615210455533937,30.31384127236931,0.644926907291912,1.736397603563502,27.520962113951,3000.0,0.652431584451804,1.6740173435593515,26.223018677187245,3003.0,2621.048262119293,3852.870769262314,2621.048262119293,1220.5474019050598,0.1391530036926269,0.0 +16500,0.1692856848239898,1.9694981575012207,,,,,,,,,,,,,,,,, +17000,0.1685598194599151,1.9850728511810305,,,,,,,,,,,,,,,,, +17500,0.1715176254510879,1.9270329475402832,,,,,,,,,,,,,,,,, +18000,0.1729084849357605,1.833079814910889,,,,,,,,,,,,,,,,, +18500,0.1577482521533966,1.950231909751892,,,,,,,,,,,,,,,,, +19000,0.1588059365749359,1.8404455184936523,,,,,,,,,,,,,,,,, +19500,0.1661652624607086,1.9476399421691888,,,,,,,,,,,,,,,,, +20000,0.1627473086118698,1.8940504789352417,,,,,,,,,,,,,,,,, +20335,,,0.6405194984153232,1.7526106259473613,31.23142393247607,0.6531599112224274,1.6666036378966165,27.66911449034538,3000.0,0.6614142118412643,1.6036165097902504,26.95614184291244,3003.0,3263.100668668747,4691.734410762787,3263.100668668747,1414.761714220047,0.1748809814453125,0.0 +20500,0.1680981963872909,1.9348467588424685,,,,,,,,,,,,,,,,, +21000,0.1530636698007583,1.8490182161331177,,,,,,,,,,,,,,,,, +21500,0.1582259535789489,1.8188978433609009,,,,,,,,,,,,,,,,, +22000,0.1701591461896896,1.901313662528992,,,,,,,,,,,,,,,,, +22500,0.1732169091701507,1.8633759021759035,,,,,,,,,,,,,,,,, +23000,0.1482104659080505,1.9576884508132928,,,,,,,,,,,,,,,,, +23500,0.1573890149593353,1.832968473434448,,,,,,,,,,,,,,,,, +24000,0.1574636250734329,1.831547737121582,,,,,,,,,,,,,,,,, +24407,,,0.6414288155870791,1.7548805759699198,31.41940241037089,0.6600042156947837,1.6223898920968125,28.151750697478555,3000.0,0.6718029167392946,1.5446280721631516,27.845316861934634,3003.0,3905.2253880500793,5528.938032865524,3905.2253880500793,1607.2352409362793,0.2161371707916259,0.0 +24500,0.2033358961343765,1.8166927099227903,,,,,,,,,,,,,,,,, +25000,0.1512181311845779,1.8649860620498653,,,,,,,,,,,,,,,,, +25500,0.2088886797428131,1.7984362840652466,,,,,,,,,,,,,,,,, +26000,0.1445168852806091,1.7345086336135864,,,,,,,,,,,,,,,,, +26500,0.1541185975074768,1.8396848440170288,,,,,,,,,,,,,,,,, +27000,0.1550447046756744,1.7563836574554443,,,,,,,,,,,,,,,,, +27500,0.1571839600801468,1.852638959884644,,,,,,,,,,,,,,,,, +28000,0.1547568142414093,1.8504327535629272,,,,,,,,,,,,,,,,, +28478,,,0.646569454394195,1.696688394033231,31.908780533204727,0.6634015697263518,1.5865189985245067,28.6387571484488,3000.0,0.6753123002730812,1.5034113357736332,28.228522569674404,3003.0,4547.240791797638,6416.162347793579,4547.240791797638,1849.8468225002289,0.2523863315582275,0.0 +28500,0.177907109260559,1.882025122642517,,,,,,,,,,,,,,,,, +29000,0.1543263345956802,1.818682551383972,,,,,,,,,,,,,,,,, +29500,0.1790771633386612,1.7921644449234009,,,,,,,,,,,,,,,,, +30000,0.1509639918804168,1.7964622974395752,,,,,,,,,,,,,,,,, +30500,0.1628822237253189,1.8199386596679688,,,,,,,,,,,,,,,,, +31000,0.1564475297927856,1.7379435300827026,,,,,,,,,,,,,,,,, +31500,0.1617218106985092,1.7690364122390747,,,,,,,,,,,,,,,,, +32000,0.1633485853672027,1.7651197910308838,,,,,,,,,,,,,,,,, +32500,0.1781674921512603,1.7224966287612915,,,,,,,,,,,,,,,,, +32551,,,0.6574741352642615,1.6197046035219578,32.82359983605892,0.6672576905432047,1.5561068097419746,29.33604280909626,3000.0,0.6806228574748707,1.476671351170763,28.605820851693665,3003.0,5189.318318843842,7264.40878868103,5189.318318843842,2053.412314414978,0.2888314723968506,0.0 +33000,0.226987674832344,1.805983304977417,,,,,,,,,,,,,,,,, +33500,0.1542650163173675,1.7637791633605957,,,,,,,,,,,,,,,,, +34000,0.1706307530403137,1.702488899230957,,,,,,,,,,,,,,,,, +34500,0.1895196139812469,1.7923831939697266,,,,,,,,,,,,,,,,, +35000,0.1699455380439758,1.7349613904953003,,,,,,,,,,,,,,,,, +35500,0.1941119879484176,1.744235277175903,,,,,,,,,,,,,,,,, +36000,0.1748156100511551,1.6796891689300537,,,,,,,,,,,,,,,,, +36500,0.181170180439949,1.692702293395996,,,,,,,,,,,,,,,,, +36623,,,0.6546037249512426,1.64831480457122,32.60188823846328,0.6693903361396635,1.538441091864949,29.129087562232385,3000.0,0.6837952472256116,1.4489703169484631,28.47953293764722,3003.0,5831.318049192429,8117.268192768097,5831.318049192429,2261.6669857501984,0.326277494430542,0.0 +37000,0.2407483905553817,1.6682945489883425,,,,,,,,,,,,,,,,, +37500,0.2123768180608749,1.7915867567062378,,,,,,,,,,,,,,,,, +38000,0.1753620654344558,1.750494122505188,,,,,,,,,,,,,,,,, +38500,0.2209336906671524,1.6957263946533203,,,,,,,,,,,,,,,,, +39000,0.182938739657402,1.617396116256714,,,,,,,,,,,,,,,,, +39500,0.1950473040342331,1.7647780179977417,,,,,,,,,,,,,,,,, +40000,0.2431928962469101,1.6916459798812866,,,,,,,,,,,,,,,,, +40500,0.2379485666751861,1.7580806016921997,,,,,,,,,,,,,,,,, +40696,,,0.6608371754979673,1.5949718509391535,32.70192794968317,0.6731596632403813,1.5138081440403717,29.136297489173543,3000.0,0.6858985532508279,1.4243017728487597,28.86010418388668,3003.0,6473.437443256378,9036.62000489235,6473.437443256378,2536.2961554527283,0.362851619720459,0.0 +41000,0.2024037986993789,1.689362645149231,,,,,,,,,,,,,,,,, +41500,0.2083912342786789,1.675607442855835,,,,,,,,,,,,,,,,, +42000,0.2740936279296875,1.7567083835601809,,,,,,,,,,,,,,,,, +42500,0.2294646352529525,1.676990270614624,,,,,,,,,,,,,,,,, +43000,0.2095404863357544,1.6317740678787231,,,,,,,,,,,,,,,,, +43500,0.2035191506147384,1.755216360092163,,,,,,,,,,,,,,,,, +44000,0.2234981209039688,1.751802921295166,,,,,,,,,,,,,,,,, +44500,0.2365189790725708,1.7099812030792236,,,,,,,,,,,,,,,,, +44769,,,0.6736368540047972,1.5027317254943593,33.463669608567024,0.67566428190599,1.4967648029782643,29.447153797348868,3000.0,0.6869095345999652,1.413147659926791,29.056115384466224,3003.0,7115.437336683273,9888.822259187698,7115.437336683273,2743.8992309570312,0.4001736640930176,0.0 +45000,0.2350603342056274,1.6828737258911133,,,,,,,,,,,,,,,,, +45500,0.2871157824993133,1.6476932764053345,,,,,,,,,,,,,,,,, +46000,0.2546157836914062,1.7120537757873535,,,,,,,,,,,,,,,,, +46500,0.2367957085371017,1.6190688610076904,,,,,,,,,,,,,,,,, +47000,0.2374975234270095,1.6768702268600464,,,,,,,,,,,,,,,,, +47500,0.3034848272800445,1.6696103811264038,,,,,,,,,,,,,,,,, +48000,0.2657664716243744,1.6031473875045776,,,,,,,,,,,,,,,,, +48500,0.2489755004644394,1.675259709358215,,,,,,,,,,,,,,,,, +48843,,,0.6602731896896611,1.5895205249471338,33.20233996811842,0.6778093266047538,1.4871544331440405,29.348687414056062,3000.0,0.690163267677648,1.3950370584219394,29.252620058582394,3003.0,7757.446277618408,10777.529091835022,7757.446277618408,2987.965854883194,0.4670050144195556,0.0 +49000,0.2374563217163086,1.6037966012954712,,,,,,,,,,,,,,,,, +49500,0.2344223111867904,1.720776081085205,,,,,,,,,,,,,,,,, +50000,0.2685203552246094,1.6080601215362549,,,,,,,,,,,,,,,,, +50500,0.2630853056907654,1.6742777824401855,,,,,,,,,,,,,,,,, +51000,0.2587460279464721,1.6935278177261353,,,,,,,,,,,,,,,,, +51500,0.2869343161582947,1.6423935890197754,,,,,,,,,,,,,,,,, +52000,0.2553168535232544,1.660990595817566,,,,,,,,,,,,,,,,, +52500,0.2836402356624603,1.624242663383484,,,,,,,,,,,,,,,,, +52918,,,0.6706479784121057,1.5236382675600897,33.46744917235679,0.6784416808223085,1.469598563563998,29.52817047409481,3000.0,0.6934518621811632,1.3803054732438558,29.445593184963908,3003.0,8399.471651315689,11709.902684926989,8399.471651315689,3275.7083344459534,0.5038421154022217,0.0 +53000,0.3086529374122619,1.5286599397659302,,,,,,,,,,,,,,,,, +53500,0.3287296593189239,1.6694175004959106,,,,,,,,,,,,,,,,, +54000,0.272642433643341,1.696863055229187,,,,,,,,,,,,,,,,, +54500,0.3063922822475433,1.625737547874451,,,,,,,,,,,,,,,,, +55000,0.3126203417778015,1.6068137884140017,,,,,,,,,,,,,,,,, +55500,0.3024630844593048,1.6142723560333252,,,,,,,,,,,,,,,,, +56000,0.264040857553482,1.647621750831604,,,,,,,,,,,,,,,,, +56500,0.3199820220470428,1.5923746824264526,,,,,,,,,,,,,,,,, +56995,,,0.6831654775287344,1.438460678152745,34.52225552751289,0.6807355147487322,1.462125585857584,29.87392002459043,3000.0,0.6953343791761083,1.3676087422288072,29.502131903352502,3003.0,9041.56005358696,12590.445948839188,9041.56005358696,3511.548393249512,0.5425922870635986,0.0 +57000,0.3063276708126068,1.58563232421875,,,,,,,,,,,,,,,,, +57500,0.3978213667869568,1.672660231590271,,,,,,,,,,,,,,,,, +58000,0.2990337610244751,1.5602748394012451,,,,,,,,,,,,,,,,, +58500,0.2906458079814911,1.6176161766052246,,,,,,,,,,,,,,,,, +59000,0.2833219468593597,1.5811725854873655,,,,,,,,,,,,,,,,, +59500,0.317019909620285,1.618083953857422,,,,,,,,,,,,,,,,, +60000,0.3074941635131836,1.6015594005584717,,,,,,,,,,,,,,,,, +60500,0.3095583617687225,1.5300050973892212,,,,,,,,,,,,,,,,, +61000,0.3066987693309784,1.6036959886550903,,,,,,,,,,,,,,,,, +61073,,,0.6728275450886868,1.5079797373218524,33.58934435863579,0.6837484966088455,1.4534882549503416,29.378392460263147,3000.0,0.6959967462669223,1.357591438614839,29.82273898744917,3003.0,9683.66823744774,13556.334055900574,9683.66823744774,3832.727107048034,0.5803112983703613,0.0 +61500,0.3111431002616882,1.5791480541229248,,,,,,,,,,,,,,,,, +62000,0.3444833755493164,1.6888129711151123,,,,,,,,,,,,,,,,, +62500,0.3032851219177246,1.6305818557739258,,,,,,,,,,,,,,,,, +63000,0.3298478424549103,1.6710878610610962,,,,,,,,,,,,,,,,, +63500,0.3126907348632812,1.611255407333374,,,,,,,,,,,,,,,,, +64000,0.3379656076431274,1.641921877861023,,,,,,,,,,,,,,,,, +64500,0.3285458385944366,1.626142144203186,,,,,,,,,,,,,,,,, +65000,0.3037836253643036,1.5925846099853516,,,,,,,,,,,,,,,,, +65146,,,0.6787732503603376,1.4658047398705072,34.23505056955816,0.6824837881737362,1.4427903482287885,29.899171017447795,3000.0,0.6989599674626692,1.3438375167044332,29.89596834098016,3003.0,10325.675132989883,14553.30824804306,10325.675132989883,4185.091699361801,0.6182959079742432,0.0 +65500,0.2964764833450317,1.5829119682312012,,,,,,,,,,,,,,,,, +66000,0.3113552033901214,1.5933730602264404,,,,,,,,,,,,,,,,, +66500,0.3676771223545074,1.6400444507598877,,,,,,,,,,,,,,,,, +67000,0.3338375389575958,1.6292152404785156,,,,,,,,,,,,,,,,, +67500,0.3184605538845062,1.5468809604644775,,,,,,,,,,,,,,,,, +68000,0.3144562244415283,1.479383111000061,,,,,,,,,,,,,,,,, +68500,0.3242087662220001,1.6114684343338013,,,,,,,,,,,,,,,,, +69000,0.3289821147918701,1.5966734886169434,,,,,,,,,,,,,,,,, +69220,,,0.6870573523560507,1.4177981832262831,34.462200462819816,0.6847900212024649,1.4348646095522684,28.746716660696627,3000.0,0.6989134855615595,1.337881478414967,29.55665481900165,3003.0,10967.742772340776,15511.32098197937,10967.742772340776,4498.430937290192,0.6577272415161133,0.0 +69500,0.3455301821231842,1.6131396293640137,,,,,,,,,,,,,,,,, +70000,0.3286156058311462,1.5600223541259766,,,,,,,,,,,,,,,,, +70500,0.3384590446949005,1.5600043535232544,,,,,,,,,,,,,,,,, +71000,0.3622392117977142,1.5559567213058472,,,,,,,,,,,,,,,,, +71500,0.3031913638114929,1.529407978057861,,,,,,,,,,,,,,,,, +72000,0.3367080390453338,1.481659173965454,,,,,,,,,,,,,,,,, +72500,0.3653931021690368,1.5306700468063354,,,,,,,,,,,,,,,,, +73000,0.3468227386474609,1.5650620460510254,,,,,,,,,,,,,,,,, +73301,,,0.6781031888779538,1.4760303899277158,34.477809736742664,0.686451500911334,1.4277885968555877,28.023441115401464,3000.0,0.7004822497240137,1.3271995380861077,28.9335588426571,3003.0,11609.769958019257,16495.714752435684,11609.769958019257,4838.17728638649,0.6972668170928955,0.0 +73500,0.2946541011333465,1.5254684686660769,,,,,,,,,,,,,,,,, +74000,0.3205336630344391,1.6093745231628418,,,,,,,,,,,,,,,,, +74500,0.3834731876850128,1.576507806777954,,,,,,,,,,,,,,,,, +75000,0.3300183415412903,1.4629385471343994,,,,,,,,,,,,,,,,, +75500,0.3497211635112762,1.5080691576004028,,,,,,,,,,,,,,,,, +76000,0.3385490775108337,1.4954978227615356,,,,,,,,,,,,,,,,, +76500,0.357876569032669,1.4921499490737915,,,,,,,,,,,,,,,,, +77000,0.3418661653995514,1.539737343788147,,,,,,,,,,,,,,,,, +77386,,,0.6885287531454164,1.4049029177343768,35.03851466280576,0.6878402003694932,1.4182358634734846,30.26158475825712,3000.0,0.7026552786008948,1.3144139285921794,30.07638390589513,3003.0,12251.764683485031,17440.068254709244,12251.764683485031,5137.922399997711,0.7374632358551025,0.0 +77500,0.3609346747398376,1.501954436302185,,,,,,,,,,,,,,,,, +78000,0.3547244071960449,1.6231677532196045,,,,,,,,,,,,,,,,, +78500,0.3534749746322632,1.498704433441162,,,,,,,,,,,,,,,,, +79000,0.387299507856369,1.5009273290634155,,,,,,,,,,,,,,,,, +79500,0.3380951881408691,1.5900403261184692,,,,,,,,,,,,,,,,, +80000,0.3353755474090576,1.4986380338668823,,,,,,,,,,,,,,,,, +80500,0.340899646282196,1.5660731792449951,,,,,,,,,,,,,,,,, +81000,0.330278605222702,1.5088577270507812,,,,,,,,,,,,,,,,, +81473,,,0.6918920154002582,1.3938208320480745,35.61374162031927,0.6870466578219737,1.4101627885891062,29.677615166629305,3000.0,0.7050723374586021,1.3048747893788857,29.86448916788208,3003.0,12893.859102725984,18372.59902620316,12893.859102725984,5425.737843036652,0.7771332263946533,0.0 +81500,0.3701776266098022,1.5012270212173462,,,,,,,,,,,,,,,,, +82000,0.3616238534450531,1.4596229791641235,,,,,,,,,,,,,,,,, +82500,0.3721172213554382,1.556060552597046,,,,,,,,,,,,,,,,, +83000,0.3568753898143768,1.4917445182800293,,,,,,,,,,,,,,,,, +83500,0.3446738421916961,1.5508875846862793,,,,,,,,,,,,,,,,, +84000,0.3349551856517792,1.5132161378860474,,,,,,,,,,,,,,,,, +84500,0.3432998657226562,1.5055902004241943,,,,,,,,,,,,,,,,, +85000,0.3426181077957153,1.4342119693756104,,,,,,,,,,,,,,,,, +85500,0.372152179479599,1.505837321281433,,,,,,,,,,,,,,,,, +85560,,,0.6885686927180703,1.4019506491952032,34.92511946075621,0.6890925097022975,1.4029911671894957,28.31170686241403,3000.0,0.7047237232002789,1.2981343690081923,30.09348926656585,3003.0,13535.897929668428,19365.742505788803,13535.897929668428,5774.228710174561,0.8166906833648682,0.0 +86000,0.3746928572654724,1.5311036109924316,,,,,,,,,,,,,,,,, +86500,0.3957862854003906,1.4946348667144775,,,,,,,,,,,,,,,,, +87000,0.3755208253860473,1.5287094116210938,,,,,,,,,,,,,,,,, +87500,0.3421024084091186,1.3604779243469238,,,,,,,,,,,,,,,,, +88000,0.3726125955581665,1.4620274305343628,,,,,,,,,,,,,,,,, +88500,0.3520002961158752,1.3840622901916504,,,,,,,,,,,,,,,,, +89000,0.362135499715805,1.374592661857605,,,,,,,,,,,,,,,,, +89500,0.406499832868576,1.4369378089904783,,,,,,,,,,,,,,,,, +89648,,,0.6970119567453876,1.347744750348643,35.79026020455463,0.6884353572801329,1.408247805358892,30.20290091938968,3000.0,0.7057695659752484,1.3033401603625587,30.257084271714383,3003.0,14178.007321834564,20242.164640188217,14178.007321834564,6005.925210952759,0.8602900505065918,0.0 +90000,0.3361519277095794,1.4988948106765747,,,,,,,,,,,,,,,,, +90500,0.3392194807529449,1.4466071128845217,,,,,,,,,,,,,,,,, +91000,0.3463259041309356,1.4886609315872192,,,,,,,,,,,,,,,,, +91500,0.3723219037055969,1.4984523057937622,,,,,,,,,,,,,,,,, +92000,0.4999997913837433,1.4862191677093506,,,,,,,,,,,,,,,,, +92500,0.344241052865982,1.4430913925170898,,,,,,,,,,,,,,,,, +93000,0.350094199180603,1.4653054475784302,,,,,,,,,,,,,,,,, +93500,0.3542776703834533,1.4757039546966553,,,,,,,,,,,,,,,,, +93735,,,0.6938509699974864,1.373772329079359,35.663830096058305,0.6909895723549615,1.393101139477502,30.513111798349964,3000.0,0.7076753239207484,1.2846936552204986,30.66273337214052,3003.0,14819.999969244003,21107.47658610344,14819.999969244003,6226.633292675018,0.8993208408355713,0.0 +94000,0.3470673263072967,1.387911081314087,,,,,,,,,,,,,,,,, +94500,0.3865800201892853,1.46714448928833,,,,,,,,,,,,,,,,, +95000,0.3855903446674347,1.49406099319458,,,,,,,,,,,,,,,,, +95500,0.3660434782505035,1.4500000476837158,,,,,,,,,,,,,,,,, +96000,0.3968791365623474,1.4460276365280151,,,,,,,,,,,,,,,,, +96500,0.3394978642463684,1.425642728805542,,,,,,,,,,,,,,,,, +97000,0.3537770807743072,1.4642565250396729,,,,,,,,,,,,,,,,, +97500,0.4223167300224304,1.4794732332229614,,,,,,,,,,,,,,,,, +97824,,,0.6972819389292072,1.3497797352115568,35.71535766677376,0.6925146619384757,1.390379052181622,29.095635009310403,3000.0,0.7093835337865319,1.2811286205043286,30.381134391662084,3003.0,15462.022831916807,22108.956053495407,15462.022831916807,6583.471929550171,0.9379317760467528,0.0 +98000,0.352455735206604,1.4723275899887085,,,,,,,,,,,,,,,,, +98500,0.349116176366806,1.3610161542892456,,,,,,,,,,,,,,,,, +99000,0.3533047437667846,1.4964324235916138,,,,,,,,,,,,,,,,, +99500,0.3453401029109955,1.3773599863052368,,,,,,,,,,,,,,,,, +100000,0.3522440493106842,1.4115097522735596,,,,,,,,,,,,,,,,, +100500,0.3511190414428711,1.436450481414795,,,,,,,,,,,,,,,,, +101000,0.3521145284175873,1.4193453788757324,,,,,,,,,,,,,,,,, +101500,0.33188596367836,1.3952628374099731,,,,,,,,,,,,,,,,, +101911,,,0.6997586999531124,1.336101624772709,36.22656518179386,0.6933578008952151,1.3865799866089694,30.279620503488072,3000.0,0.7086514438440532,1.2787330776828771,30.30002914573537,3003.0,16103.99957895279,23037.213791370392,16103.99957895279,6867.136032581329,0.9784302711486816,0.0 +102000,0.3373357057571411,1.4116942882537842,,,,,,,,,,,,,,,,, +102500,0.3305014371871948,1.418316125869751,,,,,,,,,,,,,,,,, +103000,0.3470311760902405,1.4723671674728394,,,,,,,,,,,,,,,,, +103500,0.3662768006324768,1.447679042816162,,,,,,,,,,,,,,,,, +104000,0.3278049826622009,1.332318902015686,,,,,,,,,,,,,,,,, +104500,0.364681988954544,1.4929630756378174,,,,,,,,,,,,,,,,, +105000,0.3424729406833648,1.4160815477371216,,,,,,,,,,,,,,,,, +105500,0.3379073143005371,1.4164386987686155,,,,,,,,,,,,,,,,, +105999,,,0.7005885488132634,1.3351153460701042,36.09823027073485,0.6935685856344,1.3848594174281783,30.316039877116665,3000.0,0.7098018708965197,1.27362170269014,30.5515793034812,3003.0,16746.064346313477,23946.72591495514,16746.064346313477,7131.977272987366,1.0208284854888916,0.0 +106000,0.3427518010139465,1.4317693710327148,,,,,,,,,,,,,,,,, +106500,0.3392427861690521,1.379235863685608,,,,,,,,,,,,,,,,, +107000,0.3434584140777588,1.4206600189208984,,,,,,,,,,,,,,,,, +107500,0.335109680891037,1.416383981704712,,,,,,,,,,,,,,,,, +108000,0.3683596551418304,1.4540412425994873,,,,,,,,,,,,,,,,, +108500,0.3478862643241882,1.419749736785889,,,,,,,,,,,,,,,,, +109000,0.3260056972503662,1.4650822877883911,,,,,,,,,,,,,,,,, +109500,0.3410912454128265,1.466293215751648,,,,,,,,,,,,,,,,, +110000,0.3370470404624939,1.3672044277191162,,,,,,,,,,,,,,,,, +110087,,,0.7035749922581978,1.309232314282765,36.38694155896813,0.6942009398519547,1.3826231230858883,30.652193336253045,3000.0,0.7104293765615014,1.271752258729882,30.43325747213432,3003.0,17388.03769659996,24845.13564896584,17388.03769659996,7385.803707122803,1.0621931552886963,0.0 +110500,0.3510603606700897,1.3958344459533691,,,,,,,,,,,,,,,,, +111000,0.3246383965015411,1.3198848962783811,,,,,,,,,,,,,,,,, +111500,0.3351044952869415,1.394584059715271,,,,,,,,,,,,,,,,, +112000,0.3474562764167785,1.4272499084472656,,,,,,,,,,,,,,,,, +112500,0.329690545797348,1.3887813091278076,,,,,,,,,,,,,,,,, +113000,0.3382913172245025,1.3899564743041992,,,,,,,,,,,,,,,,, +113500,0.3399111926555633,1.400858759880066,,,,,,,,,,,,,,,,, +114000,0.336271733045578,1.3695464134216309,,,,,,,,,,,,,,,,, +114173,,,0.7029496215087444,1.3174546640393583,36.05714637756056,0.693680177555145,1.3834639372109458,30.78495669829378,3000.0,0.710499099413166,1.2717316505432572,30.540808370113545,3003.0,18030.13545131684,25751.09917759896,18030.13545131684,7647.055639743805,1.104595184326172,0.0 +114500,0.3624247014522552,1.469241976737976,,,,,,,,,,,,,,,,, +115000,0.3906366527080536,1.4454654455184937,,,,,,,,,,,,,,,,, +115500,0.3397324979305267,1.4607791900634766,,,,,,,,,,,,,,,,, +116000,0.3606376349925995,1.4325138330459597,,,,,,,,,,,,,,,,, +116500,0.3280481696128845,1.3874772787094116,,,,,,,,,,,,,,,,, +117000,0.3422826826572418,1.445543885231018,,,,,,,,,,,,,,,,, +117500,0.3395417034626007,1.360105037689209,,,,,,,,,,,,,,,,, +118000,0.3275265693664551,1.3790663480758667,,,,,,,,,,,,,,,,, +118260,,,0.7059680927383081,1.3004796195497454,36.6821103814912,0.6938413658851099,1.381956768204982,30.666652184073417,3000.0,0.7104874789378886,1.270655667160537,30.5197138290683,3003.0,18672.25456643105,26684.23622441292,18672.25456643105,7935.464534521103,1.14502215385437,0.0 +118500,0.37288498878479,1.3962966203689575,,,,,,,,,,,,,,,,, +119000,0.342928409576416,1.4144917726516724,,,,,,,,,,,,,,,,, +119500,0.3472653329372406,1.3548619747161863,,,,,,,,,,,,,,,,, +120000,0.3310556411743164,1.4573118686676023,,,,,,,,,,,,,,,,, +120500,0.3335027396678924,1.4041377305984497,,,,,,,,,,,,,,,,, +121000,0.3520818650722503,1.4159877300262451,,,,,,,,,,,,,,,,, +121500,0.3451318144798279,1.382717490196228,,,,,,,,,,,,,,,,, +122000,0.3296763300895691,1.4487360715866089,,,,,,,,,,,,,,,,, +122346,,,0.6997066171583122,1.3324967338238327,36.64013427257552,0.6938537649874149,1.382162225204895,30.657045900752426,3000.0,0.7104061356109465,1.2707375551972575,30.45658956250633,3003.0,19314.32937502861,27601.87896633148,19314.32937502861,8208.439351797104,1.19000506401062,0.0 +122500,0.3244821429252624,1.360991597175598,,,,,,,,,,,,,,,,, +123000,0.3257182836532593,1.4356307983398438,,,,,,,,,,,,,,,,, +123500,0.3732545375823974,1.3895421028137207,,,,,,,,,,,,,,,,, +124000,0.3609690368175506,1.3997453451156616,,,,,,,,,,,,,,,,, +124500,0.3362762629985809,1.4128975868225098,,,,,,,,,,,,,,,,, +125000,0.3530727028846741,1.3688397407531738,,,,,,,,,,,,,,,,, +125500,0.3324375748634338,1.3806263208389282,,,,,,,,,,,,,,,,, +126000,0.3372086584568023,1.356728196144104,,,,,,,,,,,,,,,,, +126432,,,0.7062779241608431,1.2974285606885096,36.67276794079318,0.6939033613966349,1.3826583830330683,30.73521655991802,3000.0,0.7104874789378886,1.270744908779269,30.44486588346862,3003.0,19956.34680557251,28512.538029670715,19956.34680557251,8474.484276294708,1.2323086261749268,0.0 +126500,0.3191023170948028,1.4090180397033691,,,,,,,,,,,,,,,,, +127000,0.3204605877399444,1.3784387111663818,,,,,,,,,,,,,,,,, +127500,0.3418961465358734,1.41139554977417,,,,,,,,,,,,,,,,, +128000,0.350053459405899,1.4283374547958374,,,,,,,,,,,,,,,,, +128500,0.3469051420688629,1.3955767154693604,,,,,,,,,,,,,,,,, +129000,0.3347821235656738,1.4147969484329224,,,,,,,,,,,,,,,,, +129500,0.3661850988864898,1.415445327758789,,,,,,,,,,,,,,,,, +130000,0.3458209633827209,1.4175876379013062,,,,,,,,,,,,,,,,, +130500,0.350833922624588,1.4264278411865234,,,,,,,,,,,,,,,,, +130517,,,0.7066352230057698,1.2944492899418445,36.1449996008607,0.6938537649874149,1.3822447567296128,30.70360554311707,3000.0,0.7105572017895532,1.271061203590727,30.34159563464225,3003.0,20598.37356328964,29422.16049194336,20598.37356328964,8739.491944551468,1.2747008800506592,0.0 +131000,0.350923478603363,1.400408387184143,,,,,,,,,,,,,,,,, +131500,0.3676856160163879,1.374926209449768,,,,,,,,,,,,,,,,, +132000,0.3346277475357055,1.365755558013916,,,,,,,,,,,,,,,,, +132500,0.3553057610988617,1.308014988899231,,,,,,,,,,,,,,,,, +133000,0.3801806569099426,1.4990898370742798,,,,,,,,,,,,,,,,, +133500,0.3608574569225311,1.4996745586395264,,,,,,,,,,,,,,,,, +134000,0.3327431976795196,1.4202638864517212,,,,,,,,,,,,,,,,, +134500,0.3528099060058594,1.3686052560806274,,,,,,,,,,,,,,,,, +134601,,,0.7052066596013812,1.303047331715155,36.55484407583282,0.6939033613966349,1.382166681132286,30.8441882006884,3000.0,0.7101504851548428,1.2720226163500088,30.57973386492272,3003.0,21240.45601129532,30324.969521284103,21240.45601129532,8997.632767438889,1.3170082569122314,0.0 +135000,0.3527534008026123,1.3400816917419434,,,,,,,,,,,,,,,,, +135500,0.3441154956817627,1.3862758874893188,,,,,,,,,,,,,,,,, +136000,0.3574599325656891,1.3964715003967283,,,,,,,,,,,,,,,,, +136500,0.3657817840576172,1.3964041471481323,,,,,,,,,,,,,,,,, +137000,0.3766965568065643,1.4155033826828003,,,,,,,,,,,,,,,,, +137500,0.3452496528625488,1.372942328453064,,,,,,,,,,,,,,,,, +138000,0.3459999561309814,1.4092419147491455,,,,,,,,,,,,,,,,, +138500,0.3555871546268463,1.4408131837844849,,,,,,,,,,,,,,,,, +138684,,,0.7082950367647058,1.2883000990923714,36.50281435992577,0.6930230251329804,1.385253767002269,30.805347272254515,3000.0,0.7096275637673581,1.2750479344605194,30.415895458234928,3003.0,21882.56513190269,31192.38531279564,21882.56513190269,9220.341248989103,1.3608474731445312,0.0 +139000,0.358333021402359,1.36529541015625,,,,,,,,,,,,,,,,, +139500,0.3329107165336609,1.333735108375549,,,,,,,,,,,,,,,,, +140000,0.3514840602874756,1.452021837234497,,,,,,,,,,,,,,,,, +140500,0.3466958999633789,1.4294154644012451,,,,,,,,,,,,,,,,, +141000,0.3721181750297546,1.4260846376419067,,,,,,,,,,,,,,,,, +141500,0.373430073261261,1.4338263273239136,,,,,,,,,,,,,,,,, +142000,0.3630170226097107,1.4387694597244265,,,,,,,,,,,,,,,,, +142500,0.3195290267467499,1.399448037147522,,,,,,,,,,,,,,,,, +142760,,,0.7007713844923147,1.3295150991371922,36.41966519680081,0.6934693928159601,1.3867799221336374,30.73410874423248,3000.0,0.709174365231538,1.2770021715763176,30.39403426658349,3003.0,22524.6876103878,32080.60576915741,22524.6876103878,9463.840975999832,1.406548023223877,0.0 +143000,0.37263023853302,1.4334509372711182,,,,,,,,,,,,,,,,, +143500,0.3658254742622375,1.4014443159103394,,,,,,,,,,,,,,,,, +144000,0.3267988860607147,1.4154433012008667,,,,,,,,,,,,,,,,, +144500,0.3779540061950683,1.446444392204285,,,,,,,,,,,,,,,,, +145000,0.3631955981254577,1.4667593240737915,,,,,,,,,,,,,,,,, +145500,0.3479374051094055,1.4156177043914795,,,,,,,,,,,,,,,,, +146000,0.3949091136455536,1.4348268508911133,,,,,,,,,,,,,,,,, +146500,0.3632112145423889,1.3845211267471311,,,,,,,,,,,,,,,,, +146840,,,0.7025382336456082,1.3191507633860655,36.45069613420243,0.6923782718131207,1.3888317798291403,30.375924389895964,3000.0,0.7088489919237697,1.2799526828772296,30.26458936712677,3003.0,23166.677849292755,32983.62984442711,23166.677849292755,9722.284867286682,1.4494123458862305,0.0 +147000,0.3502089977264404,1.426525354385376,,,,,,,,,,,,,,,,, +147500,0.3395274877548218,1.4253389835357666,,,,,,,,,,,,,,,,, +148000,0.3494170904159546,1.4161545038223269,,,,,,,,,,,,,,,,, +148500,0.3836730420589447,1.5537689924240112,,,,,,,,,,,,,,,,, +149000,0.3544077575206756,1.4318455457687378,,,,,,,,,,,,,,,,, +149500,0.3517037332057953,1.4277478456497192,,,,,,,,,,,,,,,,, +150000,0.349827229976654,1.3803163766860962,,,,,,,,,,,,,,,,, +150500,0.3956147134304046,1.4191895723342896,,,,,,,,,,,,,,,,, +150920,,,0.7043234266735383,1.307740741138407,36.57719162324027,0.6922666798923758,1.3949919057110265,30.71555680148201,3000.0,0.7084887571901691,1.286095738190692,30.23243484427275,3003.0,23808.78427028656,33885.05792140961,23808.78427028656,9979.016785621645,1.4916207790374756,0.0 +151000,0.3686612844467163,1.4097927808761597,,,,,,,,,,,,,,,,, +151500,0.3621746897697449,1.4309711456298828,,,,,,,,,,,,,,,,, +152000,0.3632459342479706,1.3897933959960938,,,,,,,,,,,,,,,,, +152500,0.3726466596126556,1.5108855962753296,,,,,,,,,,,,,,,,, +153000,0.3592258393764496,1.4602218866348269,,,,,,,,,,,,,,,,, +153500,0.3869150280952453,1.4388750791549685,,,,,,,,,,,,,,,,, +154000,0.3855968415737152,1.453021764755249,,,,,,,,,,,,,,,,, +154500,0.3868051767349243,1.4668394327163696,,,,,,,,,,,,,,,,, +155000,0.3819617629051208,1.427568793296814,,,,,,,,,,,,,,,,, +155001,,,,,,,,,,,,,,24450.802146673203,,,,,0.0 diff --git a/logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/trial_1/meta_data_0.json b/logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/trial_1/meta_data_0.json new file mode 100644 index 00000000..66aa053f --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/trial_1/meta_data_0.json @@ -0,0 +1,71 @@ +{ + "workload.activation": "relu", + "workload.attention_temp": 1.0, + "workload.eval_batch_size": 128, + "workload.eval_period_time_sec": 644, + "workload.glu": false, + "workload.max_allowed_runtime_sec": 16114, + "workload.num_eval_train_examples": 3072, + "workload.num_test_examples": 3003, + "workload.num_train_examples": 5906184, + "workload.num_validation_examples": 3000, + "workload.pre_ln": true, + "workload.step_hint": 120000, + "workload.target_metric_name": "bleu", + "workload.test_target_value": 30.7219, + "workload.train_mean": 30.7219, + "workload.train_stddev": 30.7219, + "workload.validation_target_value": 30.8491, + "cpu.util.avg_percent_since_last": 14.2, + "cpu.freq.current": 2200.1639999999993, + "mem.total": 359053524992, + "mem.available": 349402861568, + "mem.used": 6494851072, + "mem.percent_used": 2.7, + "mem.read_bytes_since_boot": 48899112223744, + "mem.write_bytes_since_boot": 132185031680, + "net.bytes_sent_since_boot": 31242198, + "net.bytes_recv_since_boot": 31245160, + "gpu.count": 4, + "gpu.0.compute.util": 0.0, + "gpu.0.mem.util": 0.0647705078125, + "gpu.0.mem.total": 40960.0, + "gpu.0.mem.used": 2653.0, + "gpu.0.mem.free": 37674.0, + "gpu.0.temp.current": 33.0, + "gpu.1.compute.util": 0.0, + "gpu.1.mem.util": 0.0647705078125, + "gpu.1.mem.total": 40960.0, + "gpu.1.mem.used": 2653.0, + "gpu.1.mem.free": 37674.0, + "gpu.1.temp.current": 35.0, + "gpu.2.compute.util": 0.0, + "gpu.2.mem.util": 0.0647705078125, + "gpu.2.mem.total": 40960.0, + "gpu.2.mem.used": 2653.0, + "gpu.2.mem.free": 37674.0, + "gpu.2.temp.current": 33.0, + "gpu.3.compute.util": 0.0, + "gpu.3.mem.util": 0.0647705078125, + "gpu.3.mem.total": 40960.0, + "gpu.3.mem.used": 2653.0, + "gpu.3.mem.free": 37674.0, + "gpu.3.temp.current": 35.0, + "gpu.avg.compute.util": 0.0, + "gpu.avg.mem.util": 0.0647705078125, + "gpu.avg.mem.total": 40960.0, + "gpu.avg.mem.used": 2653.0, + "gpu.avg.mem.free": 37674.0, + "gpu.avg.temp.current": 34.0, + "os_platform": "Linux-6.1.0-44-cloud-amd64-x86_64-with-glibc2.31", + "python_version": "3.11.10", + "python_compiler": "GCC 9.4.0", + "git_branch": "main", + "git_commit_hash": "b21be29be0a1573fb4f78f849aea019cdb520862", + "cpu_model_name": "Intel(R) Xeon(R) CPU @ 2.20GHz", + "cpu_count": 24, + "gpu_model_name": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_driver": "550.90.12", + "rng_seed": 45840584 +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/wmt_pytorch_09-16-2026-08-57-11.log b/logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/wmt_pytorch_09-16-2026-08-57-11.log new file mode 100644 index 00000000..620c877a --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_1/wmt_pytorch/wmt_pytorch_09-16-2026-08-57-11.log @@ -0,0 +1,1258 @@ +torchrun --redirects 1:0,2:0,3:0 --standalone --nnodes=1 --nproc_per_node=4 submission_runner.py --framework=pytorch --workload=wmt --submission_path=submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py --data_dir=/data/wmt --experiment_dir=/experiment_runs --experiment_name=submissions_a100/ademamix_golden/study_1 --overwrite=True --save_checkpoints=False --rng_seed=45840584 --torch_compile=true --tuning_ruleset=self 2>&1 | tee -a /logs/wmt_pytorch_09-16-2026-08-57-11.log +W0916 08:57:12.787000 8 site-packages/torch/distributed/run.py:803] +W0916 08:57:12.787000 8 site-packages/torch/distributed/run.py:803] ***************************************** +W0916 08:57:12.787000 8 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0916 08:57:12.787000 8 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-16 08:57:14.286124: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 08:57:14.286124: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 08:57:14.286124: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 08:57:14.286124: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789549034.310691 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789549034.310691 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789549034.310691 37 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789549034.310692 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789549034.318755 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789549034.318756 37 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789549034.318756 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789549034.318764 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789549034.345212 37 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789549034.345212 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789549034.345213 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789549034.345212 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789549034.345244 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789549034.345244 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789549034.345244 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789549034.345246 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789549034.345247 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789549034.345247 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789549034.345247 37 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789549034.345248 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789549034.345249 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789549034.345249 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789549034.345250 37 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789549034.345252 37 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789549040.176466 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789549040.336196 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789549040.409024 37 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789549040.458533 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W916 08:57:24.051466499 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0916 08:57:25.498730 139681154073792 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/wmt_pytorch. +I0916 08:57:25.498724 140102352004288 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/wmt_pytorch. +I0916 08:57:25.498724 140442352403648 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/wmt_pytorch. +I0916 08:57:25.498758 139909363565760 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/wmt_pytorch. +I0916 08:57:25.523416 139681154073792 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_1/wmt_pytorch/trial_1. +I0916 08:57:25.802295 139681154073792 submission_runner.py:242] Initializing dataset. +I0916 08:57:25.802470 139681154073792 submission_runner.py:251] Initializing model. +I0916 08:57:28.802854 139681154073792 submission_runner.py:290] Performing `torch.compile`. +I0916 08:57:30.240084 139909363565760 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 08:57:30.240251 139909363565760 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 08:57:30.241054 140442352403648 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 08:57:30.241213 140442352403648 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 08:57:30.247745 140102352004288 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 08:57:30.247913 140102352004288 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 08:57:30.250754 139681154073792 submission_runner.py:294] Initializing optimizer. +I0916 08:57:30.251650 139681154073792 submission_runner.py:299] Initializing metrics bundle. +I0916 08:57:30.251809 139681154073792 submission_runner.py:321] Initializing checkpoint and logger. +I0916 08:57:30.252285 139681154073792 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_1/wmt_pytorch/trial_1/meta_data_0.json. +I0916 08:57:30.252486 139681154073792 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 08:57:30.252552 139681154073792 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 08:57:30.493698 139681154073792 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_1/wmt_pytorch/trial_1/flags_0.json. +I0916 08:57:30.536112 139681154073792 submission_runner.py:359] Starting training loop. +I0916 08:57:30.742686 139681154073792 dataset_info.py:707] Load dataset info from /data/wmt/wmt17_translate/de-en/1.0.0 +I0916 08:57:30.769265 139681154073792 dataset_info.py:793] For 'wmt17_translate/de-en/1.0.0': fields info.[splits, supervised_keys] differ on disk and in the code. Keeping the one from code. +I0916 08:57:30.771814 139681154073792 reader.py:262] Creating a tf.data.Dataset reading 16 files located in folders: /data/wmt/wmt17_translate/de-en/1.0.0. +I0916 08:57:30.831055 139681154073792 logging_logger.py:49] Constructing tf.data.Dataset wmt17_translate for split train, from /data/wmt/wmt17_translate/de-en/1.0.0 +[rank2]:W0916 08:57:32.871000 39 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank1]:W0916 08:57:32.871000 38 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank3]:W0916 08:57:32.871000 40 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank0]:W0916 08:57:33.577000 37 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +I0916 08:58:23.563475 139656330917632 logging_writer.py:48] [0] global_step=0, grad_norm=0.5, loss=11.5803 +I0916 08:58:23.595397 139681154073792 submission.py:307] 0) loss = 11.580, grad_norm = 0.500 +I0916 08:58:24.302469 139681154073792 spec.py:333] Evaluating on the training split. +I0916 08:58:24.304504 139681154073792 dataset_info.py:707] Load dataset info from /data/wmt/wmt17_translate/de-en/1.0.0 +I0916 08:58:24.306660 139681154073792 dataset_info.py:793] For 'wmt17_translate/de-en/1.0.0': fields info.[splits, supervised_keys] differ on disk and in the code. Keeping the one from code. +I0916 08:58:24.307608 139681154073792 reader.py:262] Creating a tf.data.Dataset reading 16 files located in folders: /data/wmt/wmt17_translate/de-en/1.0.0. +I0916 08:58:24.341840 139681154073792 logging_logger.py:49] Constructing tf.data.Dataset wmt17_translate for split train, from /data/wmt/wmt17_translate/de-en/1.0.0 +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +I0916 08:58:57.932818 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 09:01:14.054181 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 09:01:14.104208 139681154073792 dataset_info.py:707] Load dataset info from /data/wmt/wmt14_translate/de-en/1.0.0 +I0916 09:01:14.109878 139681154073792 dataset_info.py:793] For 'wmt14_translate/de-en/1.0.0': fields info.[splits, supervised_keys] differ on disk and in the code. Keeping the one from code. +I0916 09:01:14.110680 139681154073792 reader.py:262] Creating a tf.data.Dataset reading 1 files located in folders: /data/wmt/wmt14_translate/de-en/1.0.0. +I0916 09:01:14.147849 139681154073792 logging_logger.py:49] Constructing tf.data.Dataset wmt14_translate for split validation, from /data/wmt/wmt14_translate/de-en/1.0.0 +I0916 09:01:16.526370 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 09:03:31.886614 139681154073792 spec.py:363] Evaluating on the test split. +I0916 09:03:31.888876 139681154073792 dataset_info.py:707] Load dataset info from /data/wmt/wmt14_translate/de-en/1.0.0 +I0916 09:03:31.891167 139681154073792 dataset_info.py:793] For 'wmt14_translate/de-en/1.0.0': fields info.[splits, supervised_keys] differ on disk and in the code. Keeping the one from code. +I0916 09:03:31.891958 139681154073792 reader.py:262] Creating a tf.data.Dataset reading 1 files located in folders: /data/wmt/wmt14_translate/de-en/1.0.0. +I0916 09:03:31.928586 139681154073792 logging_logger.py:49] Constructing tf.data.Dataset wmt14_translate for split test, from /data/wmt/wmt14_translate/de-en/1.0.0 +I0916 09:03:34.277142 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 09:05:49.728163 139681154073792 submission_runner.py:516] Time since start: 499.19s, Step: 1, {'train/accuracy': 0.0005500487022288431, 'train/loss': 11.740890534578583, 'train/bleu': 0.0, 'validation/accuracy': 0.00048356498989473163, 'validation/loss': 11.741181138485574, 'validation/bleu': 0.0, 'validation/num_examples': 3000, 'test/accuracy': 0.0007088489919237697, 'test/loss': 11.744577595723666, 'test/bleu': 0.0, 'test/num_examples': 3003, 'score': 53.060325622558594, 'total_duration': 499.19220662117004, 'accumulated_submission_time': 53.060325622558594, 'accumulated_eval_time': 445.42576026916504, 'accumulated_logging_time': 0} +I0916 09:05:49.752163 139652715497216 logging_writer.py:48] [1] accumulated_eval_time=445.426, accumulated_logging_time=0, accumulated_submission_time=53.0603, global_step=1, preemption_count=0, score=53.0603, test/accuracy=0.000708849, test/bleu=0, test/loss=11.7446, test/num_examples=3003, total_duration=499.192, train/accuracy=0.000550049, train/bleu=0, train/loss=11.7409, validation/accuracy=0.000483565, validation/bleu=0, validation/loss=11.7412, validation/num_examples=3000 +I0916 09:05:50.712176 139652707104512 logging_writer.py:48] [1] global_step=1, grad_norm=0.5, loss=11.5677 +I0916 09:05:50.715172 139681154073792 submission.py:307] 1) loss = 11.568, grad_norm = 0.500 +I0916 09:05:50.886785 139652715497216 logging_writer.py:48] [2] global_step=2, grad_norm=0.5, loss=11.5776 +I0916 09:05:50.889890 139681154073792 submission.py:307] 2) loss = 11.578, grad_norm = 0.500 +I0916 09:05:51.063956 139652707104512 logging_writer.py:48] [3] global_step=3, grad_norm=0.5, loss=11.5673 +I0916 09:05:51.067015 139681154073792 submission.py:307] 3) loss = 11.567, grad_norm = 0.500 +I0916 09:05:51.238089 139652715497216 logging_writer.py:48] [4] global_step=4, grad_norm=0.5, loss=11.5443 +I0916 09:05:51.241000 139681154073792 submission.py:307] 4) loss = 11.544, grad_norm = 0.500 +I0916 09:05:51.412362 139652707104512 logging_writer.py:48] [5] global_step=5, grad_norm=0.5, loss=11.5322 +I0916 09:05:51.415340 139681154073792 submission.py:307] 5) loss = 11.532, grad_norm = 0.500 +I0916 09:05:51.587826 139652715497216 logging_writer.py:48] [6] global_step=6, grad_norm=0.5, loss=11.5068 +I0916 09:05:51.590803 139681154073792 submission.py:307] 6) loss = 11.507, grad_norm = 0.500 +I0916 09:05:51.765269 139652707104512 logging_writer.py:48] [7] global_step=7, grad_norm=0.5, loss=11.4714 +I0916 09:05:51.768295 139681154073792 submission.py:307] 7) loss = 11.471, grad_norm = 0.500 +I0916 09:05:51.940211 139652715497216 logging_writer.py:48] [8] global_step=8, grad_norm=0.5, loss=11.4327 +I0916 09:05:51.943289 139681154073792 submission.py:307] 8) loss = 11.433, grad_norm = 0.500 +I0916 09:05:52.115002 139652707104512 logging_writer.py:48] [9] global_step=9, grad_norm=0.5, loss=11.4029 +I0916 09:05:52.118096 139681154073792 submission.py:307] 9) loss = 11.403, grad_norm = 0.500 +I0916 09:05:52.289375 139652715497216 logging_writer.py:48] [10] global_step=10, grad_norm=0.5, loss=11.3596 +I0916 09:05:52.292338 139681154073792 submission.py:307] 10) loss = 11.360, grad_norm = 0.500 +I0916 09:05:52.464926 139652707104512 logging_writer.py:48] [11] global_step=11, grad_norm=0.5, loss=11.3041 +I0916 09:05:52.467931 139681154073792 submission.py:307] 11) loss = 11.304, grad_norm = 0.500 +I0916 09:05:52.639128 139652715497216 logging_writer.py:48] [12] global_step=12, grad_norm=0.5, loss=11.2544 +I0916 09:05:52.642107 139681154073792 submission.py:307] 12) loss = 11.254, grad_norm = 0.500 +I0916 09:05:52.813819 139652707104512 logging_writer.py:48] [13] global_step=13, grad_norm=0.5, loss=11.2049 +I0916 09:05:52.816836 139681154073792 submission.py:307] 13) loss = 11.205, grad_norm = 0.500 +I0916 09:05:52.988936 139652715497216 logging_writer.py:48] [14] global_step=14, grad_norm=0.5, loss=11.1376 +I0916 09:05:52.991893 139681154073792 submission.py:307] 14) loss = 11.138, grad_norm = 0.500 +I0916 09:05:53.163170 139652707104512 logging_writer.py:48] [15] global_step=15, grad_norm=0.5, loss=11.0762 +I0916 09:05:53.166140 139681154073792 submission.py:307] 15) loss = 11.076, grad_norm = 0.500 +I0916 09:05:53.337704 139652715497216 logging_writer.py:48] [16] global_step=16, grad_norm=0.5, loss=11.0312 +I0916 09:05:53.340672 139681154073792 submission.py:307] 16) loss = 11.031, grad_norm = 0.500 +I0916 09:05:53.512365 139652707104512 logging_writer.py:48] [17] global_step=17, grad_norm=0.5, loss=10.9593 +I0916 09:05:53.515415 139681154073792 submission.py:307] 17) loss = 10.959, grad_norm = 0.500 +I0916 09:05:53.686578 139652715497216 logging_writer.py:48] [18] global_step=18, grad_norm=0.5, loss=10.8955 +I0916 09:05:53.689522 139681154073792 submission.py:307] 18) loss = 10.896, grad_norm = 0.500 +I0916 09:05:53.860988 139652707104512 logging_writer.py:48] [19] global_step=19, grad_norm=0.5, loss=10.8417 +I0916 09:05:53.864022 139681154073792 submission.py:307] 19) loss = 10.842, grad_norm = 0.500 +I0916 09:05:54.034947 139652715497216 logging_writer.py:48] [20] global_step=20, grad_norm=0.5, loss=10.7633 +I0916 09:05:54.037873 139681154073792 submission.py:307] 20) loss = 10.763, grad_norm = 0.500 +I0916 09:05:54.210641 139652707104512 logging_writer.py:48] [21] global_step=21, grad_norm=0.5, loss=10.6916 +I0916 09:05:54.213712 139681154073792 submission.py:307] 21) loss = 10.692, grad_norm = 0.500 +I0916 09:05:54.385926 139652715497216 logging_writer.py:48] [22] global_step=22, grad_norm=0.5, loss=10.6283 +I0916 09:05:54.388897 139681154073792 submission.py:307] 22) loss = 10.628, grad_norm = 0.500 +I0916 09:05:54.560399 139652707104512 logging_writer.py:48] [23] global_step=23, grad_norm=0.5, loss=10.5527 +I0916 09:05:54.563399 139681154073792 submission.py:307] 23) loss = 10.553, grad_norm = 0.500 +I0916 09:05:54.735030 139652715497216 logging_writer.py:48] [24] global_step=24, grad_norm=0.5, loss=10.4781 +I0916 09:05:54.738077 139681154073792 submission.py:307] 24) loss = 10.478, grad_norm = 0.500 +I0916 09:05:54.909830 139652707104512 logging_writer.py:48] [25] global_step=25, grad_norm=0.5, loss=10.4034 +I0916 09:05:54.912763 139681154073792 submission.py:307] 25) loss = 10.403, grad_norm = 0.500 +I0916 09:05:55.083951 139652715497216 logging_writer.py:48] [26] global_step=26, grad_norm=0.5, loss=10.3444 +I0916 09:05:55.086973 139681154073792 submission.py:307] 26) loss = 10.344, grad_norm = 0.500 +I0916 09:05:55.259466 139652707104512 logging_writer.py:48] [27] global_step=27, grad_norm=0.5, loss=10.2722 +I0916 09:05:55.262461 139681154073792 submission.py:307] 27) loss = 10.272, grad_norm = 0.500 +I0916 09:05:55.434068 139652715497216 logging_writer.py:48] [28] global_step=28, grad_norm=0.5, loss=10.2029 +I0916 09:05:55.437042 139681154073792 submission.py:307] 28) loss = 10.203, grad_norm = 0.500 +I0916 09:05:55.609408 139652707104512 logging_writer.py:48] [29] global_step=29, grad_norm=0.5, loss=10.1402 +I0916 09:05:55.612477 139681154073792 submission.py:307] 29) loss = 10.140, grad_norm = 0.500 +I0916 09:05:55.784267 139652715497216 logging_writer.py:48] [30] global_step=30, grad_norm=0.5, loss=10.0438 +I0916 09:05:55.787315 139681154073792 submission.py:307] 30) loss = 10.044, grad_norm = 0.500 +I0916 09:05:55.959415 139652707104512 logging_writer.py:48] [31] global_step=31, grad_norm=0.5, loss=9.99541 +I0916 09:05:55.962439 139681154073792 submission.py:307] 31) loss = 9.995, grad_norm = 0.500 +I0916 09:05:56.134021 139652715497216 logging_writer.py:48] [32] global_step=32, grad_norm=0.5, loss=9.92104 +I0916 09:05:56.137049 139681154073792 submission.py:307] 32) loss = 9.921, grad_norm = 0.500 +I0916 09:05:56.309248 139652707104512 logging_writer.py:48] [33] global_step=33, grad_norm=0.5, loss=9.86875 +I0916 09:05:56.312309 139681154073792 submission.py:307] 33) loss = 9.869, grad_norm = 0.500 +I0916 09:05:56.484021 139652715497216 logging_writer.py:48] [34] global_step=34, grad_norm=0.5, loss=9.80676 +I0916 09:05:56.486978 139681154073792 submission.py:307] 34) loss = 9.807, grad_norm = 0.500 +I0916 09:05:56.659869 139652707104512 logging_writer.py:48] [35] global_step=35, grad_norm=0.5, loss=9.74323 +I0916 09:05:56.662940 139681154073792 submission.py:307] 35) loss = 9.743, grad_norm = 0.500 +I0916 09:05:56.833919 139652715497216 logging_writer.py:48] [36] global_step=36, grad_norm=0.5, loss=9.70046 +I0916 09:05:56.836971 139681154073792 submission.py:307] 36) loss = 9.700, grad_norm = 0.500 +I0916 09:05:57.009112 139652707104512 logging_writer.py:48] [37] global_step=37, grad_norm=0.5, loss=9.64268 +I0916 09:05:57.012154 139681154073792 submission.py:307] 37) loss = 9.643, grad_norm = 0.500 +I0916 09:05:57.184575 139652715497216 logging_writer.py:48] [38] global_step=38, grad_norm=0.5, loss=9.58166 +I0916 09:05:57.187555 139681154073792 submission.py:307] 38) loss = 9.582, grad_norm = 0.500 +I0916 09:05:57.359428 139652707104512 logging_writer.py:48] [39] global_step=39, grad_norm=0.5, loss=9.51001 +I0916 09:05:57.362516 139681154073792 submission.py:307] 39) loss = 9.510, grad_norm = 0.500 +I0916 09:05:57.533799 139652715497216 logging_writer.py:48] [40] global_step=40, grad_norm=0.5, loss=9.48901 +I0916 09:05:57.536877 139681154073792 submission.py:307] 40) loss = 9.489, grad_norm = 0.500 +I0916 09:05:57.708798 139652707104512 logging_writer.py:48] [41] global_step=41, grad_norm=0.5, loss=9.41715 +I0916 09:05:57.711771 139681154073792 submission.py:307] 41) loss = 9.417, grad_norm = 0.500 +I0916 09:05:57.883285 139652715497216 logging_writer.py:48] [42] global_step=42, grad_norm=0.5, loss=9.37728 +I0916 09:05:57.886369 139681154073792 submission.py:307] 42) loss = 9.377, grad_norm = 0.500 +I0916 09:05:58.058880 139652707104512 logging_writer.py:48] [43] global_step=43, grad_norm=0.5, loss=9.30039 +I0916 09:05:58.062011 139681154073792 submission.py:307] 43) loss = 9.300, grad_norm = 0.500 +I0916 09:05:58.233787 139652715497216 logging_writer.py:48] [44] global_step=44, grad_norm=0.5, loss=9.28527 +I0916 09:05:58.236831 139681154073792 submission.py:307] 44) loss = 9.285, grad_norm = 0.500 +I0916 09:05:58.409143 139652707104512 logging_writer.py:48] [45] global_step=45, grad_norm=0.5, loss=9.21321 +I0916 09:05:58.412220 139681154073792 submission.py:307] 45) loss = 9.213, grad_norm = 0.500 +I0916 09:05:58.584578 139652715497216 logging_writer.py:48] [46] global_step=46, grad_norm=0.499999, loss=9.18139 +I0916 09:05:58.587636 139681154073792 submission.py:307] 46) loss = 9.181, grad_norm = 0.500 +I0916 09:05:58.760493 139652707104512 logging_writer.py:48] [47] global_step=47, grad_norm=0.499999, loss=9.16053 +I0916 09:05:58.763509 139681154073792 submission.py:307] 47) loss = 9.161, grad_norm = 0.500 +I0916 09:05:58.935659 139652715497216 logging_writer.py:48] [48] global_step=48, grad_norm=0.499999, loss=9.11646 +I0916 09:05:58.938618 139681154073792 submission.py:307] 48) loss = 9.116, grad_norm = 0.500 +I0916 09:05:59.110378 139652707104512 logging_writer.py:48] [49] global_step=49, grad_norm=0.499999, loss=9.04858 +I0916 09:05:59.113366 139681154073792 submission.py:307] 49) loss = 9.049, grad_norm = 0.500 +I0916 09:05:59.285920 139652715497216 logging_writer.py:48] [50] global_step=50, grad_norm=0.499999, loss=9.03951 +I0916 09:05:59.288841 139681154073792 submission.py:307] 50) loss = 9.040, grad_norm = 0.500 +I0916 09:05:59.460888 139652707104512 logging_writer.py:48] [51] global_step=51, grad_norm=0.499999, loss=8.98466 +I0916 09:05:59.463919 139681154073792 submission.py:307] 51) loss = 8.985, grad_norm = 0.500 +I0916 09:05:59.636358 139652715497216 logging_writer.py:48] [52] global_step=52, grad_norm=0.499999, loss=8.95294 +I0916 09:05:59.639400 139681154073792 submission.py:307] 52) loss = 8.953, grad_norm = 0.500 +I0916 09:05:59.822580 139652707104512 logging_writer.py:48] [53] global_step=53, grad_norm=0.499999, loss=8.93758 +I0916 09:05:59.825593 139681154073792 submission.py:307] 53) loss = 8.938, grad_norm = 0.500 +I0916 09:05:59.997946 139652715497216 logging_writer.py:48] [54] global_step=54, grad_norm=0.499999, loss=8.91681 +I0916 09:06:00.001080 139681154073792 submission.py:307] 54) loss = 8.917, grad_norm = 0.500 +I0916 09:06:00.173258 139652707104512 logging_writer.py:48] [55] global_step=55, grad_norm=0.499999, loss=8.90247 +I0916 09:06:00.176262 139681154073792 submission.py:307] 55) loss = 8.902, grad_norm = 0.500 +I0916 09:06:00.348183 139652715497216 logging_writer.py:48] [56] global_step=56, grad_norm=0.499999, loss=8.89263 +I0916 09:06:00.351243 139681154073792 submission.py:307] 56) loss = 8.893, grad_norm = 0.500 +I0916 09:06:00.524232 139652707104512 logging_writer.py:48] [57] global_step=57, grad_norm=0.499999, loss=8.82903 +I0916 09:06:00.527384 139681154073792 submission.py:307] 57) loss = 8.829, grad_norm = 0.500 +I0916 09:06:00.700633 139652715497216 logging_writer.py:48] [58] global_step=58, grad_norm=0.484025, loss=8.80542 +I0916 09:06:00.703833 139681154073792 submission.py:307] 58) loss = 8.805, grad_norm = 0.484 +I0916 09:06:00.876534 139652707104512 logging_writer.py:48] [59] global_step=59, grad_norm=0.47154, loss=8.78859 +I0916 09:06:00.879665 139681154073792 submission.py:307] 59) loss = 8.789, grad_norm = 0.472 +I0916 09:06:01.055693 139652715497216 logging_writer.py:48] [60] global_step=60, grad_norm=0.447909, loss=8.7827 +I0916 09:06:01.058815 139681154073792 submission.py:307] 60) loss = 8.783, grad_norm = 0.448 +I0916 09:06:01.230667 139652707104512 logging_writer.py:48] [61] global_step=61, grad_norm=0.426025, loss=8.73132 +I0916 09:06:01.233717 139681154073792 submission.py:307] 61) loss = 8.731, grad_norm = 0.426 +I0916 09:06:01.406193 139652715497216 logging_writer.py:48] [62] global_step=62, grad_norm=0.397381, loss=8.7427 +I0916 09:06:01.409176 139681154073792 submission.py:307] 62) loss = 8.743, grad_norm = 0.397 +I0916 09:06:01.582031 139652707104512 logging_writer.py:48] [63] global_step=63, grad_norm=0.387759, loss=8.69425 +I0916 09:06:01.585086 139681154073792 submission.py:307] 63) loss = 8.694, grad_norm = 0.388 +I0916 09:06:01.756838 139652715497216 logging_writer.py:48] [64] global_step=64, grad_norm=0.358227, loss=8.71724 +I0916 09:06:01.759840 139681154073792 submission.py:307] 64) loss = 8.717, grad_norm = 0.358 +I0916 09:06:01.932350 139652707104512 logging_writer.py:48] [65] global_step=65, grad_norm=0.336959, loss=8.67442 +I0916 09:06:01.935403 139681154073792 submission.py:307] 65) loss = 8.674, grad_norm = 0.337 +I0916 09:06:02.108026 139652715497216 logging_writer.py:48] [66] global_step=66, grad_norm=0.331202, loss=8.65828 +I0916 09:06:02.111038 139681154073792 submission.py:307] 66) loss = 8.658, grad_norm = 0.331 +I0916 09:06:02.283620 139652707104512 logging_writer.py:48] [67] global_step=67, grad_norm=0.324327, loss=8.61693 +I0916 09:06:02.286654 139681154073792 submission.py:307] 67) loss = 8.617, grad_norm = 0.324 +I0916 09:06:02.458890 139652715497216 logging_writer.py:48] [68] global_step=68, grad_norm=0.302241, loss=8.62463 +I0916 09:06:02.461994 139681154073792 submission.py:307] 68) loss = 8.625, grad_norm = 0.302 +I0916 09:06:02.634474 139652707104512 logging_writer.py:48] [69] global_step=69, grad_norm=0.290193, loss=8.61702 +I0916 09:06:02.637528 139681154073792 submission.py:307] 69) loss = 8.617, grad_norm = 0.290 +I0916 09:06:02.809891 139652715497216 logging_writer.py:48] [70] global_step=70, grad_norm=0.274812, loss=8.60777 +I0916 09:06:02.812921 139681154073792 submission.py:307] 70) loss = 8.608, grad_norm = 0.275 +I0916 09:06:02.986251 139652707104512 logging_writer.py:48] [71] global_step=71, grad_norm=0.284924, loss=8.5891 +I0916 09:06:02.989386 139681154073792 submission.py:307] 71) loss = 8.589, grad_norm = 0.285 +I0916 09:06:03.161262 139652715497216 logging_writer.py:48] [72] global_step=72, grad_norm=0.273523, loss=8.57094 +I0916 09:06:03.164325 139681154073792 submission.py:307] 72) loss = 8.571, grad_norm = 0.274 +I0916 09:06:03.336704 139652707104512 logging_writer.py:48] [73] global_step=73, grad_norm=0.254751, loss=8.56735 +I0916 09:06:03.339729 139681154073792 submission.py:307] 73) loss = 8.567, grad_norm = 0.255 +I0916 09:06:03.512262 139652715497216 logging_writer.py:48] [74] global_step=74, grad_norm=0.255528, loss=8.56057 +I0916 09:06:03.515277 139681154073792 submission.py:307] 74) loss = 8.561, grad_norm = 0.256 +I0916 09:06:03.687537 139652707104512 logging_writer.py:48] [75] global_step=75, grad_norm=0.243686, loss=8.56512 +I0916 09:06:03.690681 139681154073792 submission.py:307] 75) loss = 8.565, grad_norm = 0.244 +I0916 09:06:03.863727 139652715497216 logging_writer.py:48] [76] global_step=76, grad_norm=0.245775, loss=8.52251 +I0916 09:06:03.866763 139681154073792 submission.py:307] 76) loss = 8.523, grad_norm = 0.246 +I0916 09:06:04.039806 139652707104512 logging_writer.py:48] [77] global_step=77, grad_norm=0.234155, loss=8.51801 +I0916 09:06:04.042940 139681154073792 submission.py:307] 77) loss = 8.518, grad_norm = 0.234 +I0916 09:06:04.215452 139652715497216 logging_writer.py:48] [78] global_step=78, grad_norm=0.230207, loss=8.53604 +I0916 09:06:04.218465 139681154073792 submission.py:307] 78) loss = 8.536, grad_norm = 0.230 +I0916 09:06:04.391718 139652707104512 logging_writer.py:48] [79] global_step=79, grad_norm=0.238301, loss=8.54019 +I0916 09:06:04.394741 139681154073792 submission.py:307] 79) loss = 8.540, grad_norm = 0.238 +I0916 09:06:04.567285 139652715497216 logging_writer.py:48] [80] global_step=80, grad_norm=0.227869, loss=8.51799 +I0916 09:06:04.570336 139681154073792 submission.py:307] 80) loss = 8.518, grad_norm = 0.228 +I0916 09:06:04.744220 139652707104512 logging_writer.py:48] [81] global_step=81, grad_norm=0.230711, loss=8.47553 +I0916 09:06:04.747293 139681154073792 submission.py:307] 81) loss = 8.476, grad_norm = 0.231 +I0916 09:06:04.919825 139652715497216 logging_writer.py:48] [82] global_step=82, grad_norm=0.22396, loss=8.48433 +I0916 09:06:04.922885 139681154073792 submission.py:307] 82) loss = 8.484, grad_norm = 0.224 +I0916 09:06:05.095872 139652707104512 logging_writer.py:48] [83] global_step=83, grad_norm=0.216979, loss=8.51486 +I0916 09:06:05.098908 139681154073792 submission.py:307] 83) loss = 8.515, grad_norm = 0.217 +I0916 09:06:05.271268 139652715497216 logging_writer.py:48] [84] global_step=84, grad_norm=0.21658, loss=8.4898 +I0916 09:06:05.274387 139681154073792 submission.py:307] 84) loss = 8.490, grad_norm = 0.217 +I0916 09:06:05.447129 139652707104512 logging_writer.py:48] [85] global_step=85, grad_norm=0.216634, loss=8.46441 +I0916 09:06:05.450183 139681154073792 submission.py:307] 85) loss = 8.464, grad_norm = 0.217 +I0916 09:06:05.623292 139652715497216 logging_writer.py:48] [86] global_step=86, grad_norm=0.214494, loss=8.46013 +I0916 09:06:05.626459 139681154073792 submission.py:307] 86) loss = 8.460, grad_norm = 0.214 +I0916 09:06:05.798859 139652707104512 logging_writer.py:48] [87] global_step=87, grad_norm=0.218247, loss=8.424 +I0916 09:06:05.802014 139681154073792 submission.py:307] 87) loss = 8.424, grad_norm = 0.218 +I0916 09:06:05.974250 139652715497216 logging_writer.py:48] [88] global_step=88, grad_norm=0.218854, loss=8.42217 +I0916 09:06:05.977298 139681154073792 submission.py:307] 88) loss = 8.422, grad_norm = 0.219 +I0916 09:06:06.149810 139652707104512 logging_writer.py:48] [89] global_step=89, grad_norm=0.218676, loss=8.47174 +I0916 09:06:06.152831 139681154073792 submission.py:307] 89) loss = 8.472, grad_norm = 0.219 +I0916 09:06:06.325362 139652715497216 logging_writer.py:48] [90] global_step=90, grad_norm=0.211776, loss=8.43105 +I0916 09:06:06.328327 139681154073792 submission.py:307] 90) loss = 8.431, grad_norm = 0.212 +I0916 09:06:06.500918 139652707104512 logging_writer.py:48] [91] global_step=91, grad_norm=0.213941, loss=8.44563 +I0916 09:06:06.504020 139681154073792 submission.py:307] 91) loss = 8.446, grad_norm = 0.214 +I0916 09:06:06.676752 139652715497216 logging_writer.py:48] [92] global_step=92, grad_norm=0.223957, loss=8.44704 +I0916 09:06:06.679725 139681154073792 submission.py:307] 92) loss = 8.447, grad_norm = 0.224 +I0916 09:06:06.852713 139652707104512 logging_writer.py:48] [93] global_step=93, grad_norm=0.214437, loss=8.41053 +I0916 09:06:06.855669 139681154073792 submission.py:307] 93) loss = 8.411, grad_norm = 0.214 +I0916 09:06:07.029719 139652715497216 logging_writer.py:48] [94] global_step=94, grad_norm=0.220372, loss=8.41511 +I0916 09:06:07.032689 139681154073792 submission.py:307] 94) loss = 8.415, grad_norm = 0.220 +I0916 09:06:07.205227 139652707104512 logging_writer.py:48] [95] global_step=95, grad_norm=0.2184, loss=8.39423 +I0916 09:06:07.208258 139681154073792 submission.py:307] 95) loss = 8.394, grad_norm = 0.218 +I0916 09:06:07.381183 139652715497216 logging_writer.py:48] [96] global_step=96, grad_norm=0.211919, loss=8.35962 +I0916 09:06:07.384199 139681154073792 submission.py:307] 96) loss = 8.360, grad_norm = 0.212 +I0916 09:06:07.557313 139652707104512 logging_writer.py:48] [97] global_step=97, grad_norm=0.208442, loss=8.40184 +I0916 09:06:07.560273 139681154073792 submission.py:307] 97) loss = 8.402, grad_norm = 0.208 +I0916 09:06:07.733379 139652715497216 logging_writer.py:48] [98] global_step=98, grad_norm=0.209562, loss=8.38605 +I0916 09:06:07.736354 139681154073792 submission.py:307] 98) loss = 8.386, grad_norm = 0.210 +I0916 09:06:07.909145 139652707104512 logging_writer.py:48] [99] global_step=99, grad_norm=0.204526, loss=8.40228 +I0916 09:06:07.912151 139681154073792 submission.py:307] 99) loss = 8.402, grad_norm = 0.205 +I0916 09:06:08.085044 139652715497216 logging_writer.py:48] [100] global_step=100, grad_norm=0.215481, loss=8.36179 +I0916 09:06:08.088112 139681154073792 submission.py:307] 100) loss = 8.362, grad_norm = 0.215 +I0916 09:07:11.176946 139652707104512 logging_writer.py:48] [500] global_step=500, grad_norm=0.499999, loss=6.75434 +I0916 09:07:11.180171 139681154073792 submission.py:307] 500) loss = 6.754, grad_norm = 0.500 +I0916 09:08:30.216499 139652715497216 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.499999, loss=5.51075 +I0916 09:08:30.219952 139681154073792 submission.py:307] 1000) loss = 5.511, grad_norm = 0.500 +I0916 09:09:49.276899 139652707104512 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.499999, loss=4.60362 +I0916 09:09:49.280390 139681154073792 submission.py:307] 1500) loss = 4.604, grad_norm = 0.500 +I0916 09:11:08.400460 139652715497216 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.499999, loss=3.88147 +I0916 09:11:08.403800 139681154073792 submission.py:307] 2000) loss = 3.881, grad_norm = 0.500 +I0916 09:12:27.562152 139652707104512 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.499999, loss=3.46648 +I0916 09:12:27.565752 139681154073792 submission.py:307] 2500) loss = 3.466, grad_norm = 0.500 +I0916 09:13:46.728524 139652715497216 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.499999, loss=3.12037 +I0916 09:13:46.731718 139681154073792 submission.py:307] 3000) loss = 3.120, grad_norm = 0.500 +I0916 09:15:05.907278 139652707104512 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.411572, loss=2.88635 +I0916 09:15:05.910511 139681154073792 submission.py:307] 3500) loss = 2.886, grad_norm = 0.412 +I0916 09:16:25.063953 139652715497216 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.397095, loss=2.79428 +I0916 09:16:25.067209 139681154073792 submission.py:307] 4000) loss = 2.794, grad_norm = 0.397 +I0916 09:16:34.358075 139681154073792 spec.py:333] Evaluating on the training split. +I0916 09:16:36.593190 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 09:17:43.216036 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 09:17:45.437851 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 09:18:51.460918 139681154073792 spec.py:363] Evaluating on the test split. +I0916 09:18:53.679908 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 09:19:56.798811 139681154073792 submission_runner.py:516] Time since start: 1346.26s, Step: 4056, {'train/accuracy': 0.5461362597165066, 'train/loss': 2.615983832018747, 'train/bleu': 24.722224184660156, 'validation/accuracy': 0.5493918240319401, 'validation/loss': 2.567170586849512, 'validation/bleu': 20.598851036749906, 'validation/num_examples': 3000, 'test/accuracy': 0.5522630875602812, 'test/loss': 2.5806624397187843, 'test/bleu': 19.29351834035134, 'test/num_examples': 3003, 'score': 694.9455635547638, 'total_duration': 1346.2628633975983, 'accumulated_submission_time': 694.9455635547638, 'accumulated_eval_time': 647.8665144443512, 'accumulated_logging_time': 0.033881425857543945} +I0916 09:19:56.823670 139652707104512 logging_writer.py:48] [4056] accumulated_eval_time=647.867, accumulated_logging_time=0.0338814, accumulated_submission_time=694.946, global_step=4056, preemption_count=0, score=694.946, test/accuracy=0.552263, test/bleu=19.2935, test/loss=2.58066, test/num_examples=3003, total_duration=1346.26, train/accuracy=0.546136, train/bleu=24.7222, train/loss=2.61598, validation/accuracy=0.549392, validation/bleu=20.5989, validation/loss=2.56717, validation/num_examples=3000 +I0916 09:21:07.600837 139652715497216 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.36344, loss=2.64548 +I0916 09:21:07.604231 139681154073792 submission.py:307] 4500) loss = 2.645, grad_norm = 0.363 +I0916 09:22:26.601651 139652707104512 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.3162, loss=2.51292 +I0916 09:22:26.604821 139681154073792 submission.py:307] 5000) loss = 2.513, grad_norm = 0.316 +I0916 09:23:45.748028 139652715497216 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.31965, loss=2.41601 +I0916 09:23:45.751485 139681154073792 submission.py:307] 5500) loss = 2.416, grad_norm = 0.320 +I0916 09:25:04.906602 139652707104512 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.302797, loss=2.40354 +I0916 09:25:04.909891 139681154073792 submission.py:307] 6000) loss = 2.404, grad_norm = 0.303 +I0916 09:26:24.003509 139652715497216 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.308596, loss=2.35358 +I0916 09:26:24.006794 139681154073792 submission.py:307] 6500) loss = 2.354, grad_norm = 0.309 +I0916 09:27:43.131821 139652707104512 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.282544, loss=2.38677 +I0916 09:27:43.135071 139681154073792 submission.py:307] 7000) loss = 2.387, grad_norm = 0.283 +I0916 09:29:02.269595 139652715497216 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.274571, loss=2.29541 +I0916 09:29:02.273073 139681154073792 submission.py:307] 7500) loss = 2.295, grad_norm = 0.275 +I0916 09:30:21.383759 139652707104512 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.25458, loss=2.3343 +I0916 09:30:21.387007 139681154073792 submission.py:307] 8000) loss = 2.334, grad_norm = 0.255 +I0916 09:30:41.419931 139681154073792 spec.py:333] Evaluating on the training split. +I0916 09:30:43.654655 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 09:31:55.784364 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 09:31:58.001111 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 09:33:01.470072 139681154073792 spec.py:363] Evaluating on the test split. +I0916 09:33:03.688248 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 09:34:01.884738 139681154073792 submission_runner.py:516] Time since start: 2191.35s, Step: 8124, {'train/accuracy': 0.5977578372947919, 'train/loss': 2.1133938968422497, 'train/bleu': 28.960536769195457, 'validation/accuracy': 0.6121684790021202, 'validation/loss': 2.0209432462089745, 'validation/bleu': 24.8842243333632, 'validation/num_examples': 3000, 'test/accuracy': 0.6148044855034571, 'test/loss': 1.9902086601592006, 'test/bleu': 23.51979872328674, 'test/num_examples': 3003, 'score': 1336.9580190181732, 'total_duration': 2191.3487401008606, 'accumulated_submission_time': 1336.9580190181732, 'accumulated_eval_time': 848.3313324451447, 'accumulated_logging_time': 0.06844353675842285} +I0916 09:34:01.911160 139652715497216 logging_writer.py:48] [8124] accumulated_eval_time=848.331, accumulated_logging_time=0.0684435, accumulated_submission_time=1336.96, global_step=8124, preemption_count=0, score=1336.96, test/accuracy=0.614804, test/bleu=23.5198, test/loss=1.99021, test/num_examples=3003, total_duration=2191.35, train/accuracy=0.597758, train/bleu=28.9605, train/loss=2.11339, validation/accuracy=0.612168, validation/bleu=24.8842, validation/loss=2.02094, validation/num_examples=3000 +I0916 09:35:01.939225 139652707104512 logging_writer.py:48] [8500] global_step=8500, grad_norm=0.240838, loss=2.19863 +I0916 09:35:01.942476 139681154073792 submission.py:307] 8500) loss = 2.199, grad_norm = 0.241 +I0916 09:36:20.874454 139652715497216 logging_writer.py:48] [9000] global_step=9000, grad_norm=0.239872, loss=2.14106 +I0916 09:36:20.877654 139681154073792 submission.py:307] 9000) loss = 2.141, grad_norm = 0.240 +I0916 09:37:39.962931 139652707104512 logging_writer.py:48] [9500] global_step=9500, grad_norm=0.253489, loss=2.16684 +I0916 09:37:39.966379 139681154073792 submission.py:307] 9500) loss = 2.167, grad_norm = 0.253 +I0916 09:38:59.119316 139652715497216 logging_writer.py:48] [10000] global_step=10000, grad_norm=0.221962, loss=2.07576 +I0916 09:38:59.122517 139681154073792 submission.py:307] 10000) loss = 2.076, grad_norm = 0.222 +I0916 09:40:18.260544 139652707104512 logging_writer.py:48] [10500] global_step=10500, grad_norm=0.213502, loss=2.04441 +I0916 09:40:18.263937 139681154073792 submission.py:307] 10500) loss = 2.044, grad_norm = 0.214 +I0916 09:41:37.424098 139652715497216 logging_writer.py:48] [11000] global_step=11000, grad_norm=0.220687, loss=2.10975 +I0916 09:41:37.427347 139681154073792 submission.py:307] 11000) loss = 2.110, grad_norm = 0.221 +I0916 09:42:56.564673 139652707104512 logging_writer.py:48] [11500] global_step=11500, grad_norm=0.209134, loss=2.05962 +I0916 09:42:56.568239 139681154073792 submission.py:307] 11500) loss = 2.060, grad_norm = 0.209 +I0916 09:44:15.678075 139652715497216 logging_writer.py:48] [12000] global_step=12000, grad_norm=0.206218, loss=2.17938 +I0916 09:44:15.681627 139681154073792 submission.py:307] 12000) loss = 2.179, grad_norm = 0.206 +I0916 09:44:46.498436 139681154073792 spec.py:333] Evaluating on the training split. +I0916 09:44:48.730585 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 09:45:55.010261 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 09:45:57.223760 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 09:47:01.819065 139681154073792 spec.py:363] Evaluating on the test split. +I0916 09:47:04.038912 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 09:48:02.558714 139681154073792 submission_runner.py:516] Time since start: 3032.02s, Step: 12192, {'train/accuracy': 0.6143790849673203, 'train/loss': 1.976550205612415, 'train/bleu': 29.92785405711678, 'validation/accuracy': 0.6319698453831942, 'validation/loss': 1.8417314648919418, 'validation/bleu': 26.584652851785467, 'validation/num_examples': 3000, 'test/accuracy': 0.6400325373307768, 'test/loss': 1.783123801638487, 'test/bleu': 25.529453286545973, 'test/num_examples': 3003, 'score': 1978.9634940624237, 'total_duration': 3032.022760629654, 'accumulated_submission_time': 1978.9634940624237, 'accumulated_eval_time': 1044.3916444778442, 'accumulated_logging_time': 0.10451340675354004} +I0916 09:48:02.583662 139652707104512 logging_writer.py:48] [12192] accumulated_eval_time=1044.39, accumulated_logging_time=0.104513, accumulated_submission_time=1978.96, global_step=12192, preemption_count=0, score=1978.96, test/accuracy=0.640033, test/bleu=25.5295, test/loss=1.78312, test/num_examples=3003, total_duration=3032.02, train/accuracy=0.614379, train/bleu=29.9279, train/loss=1.97655, validation/accuracy=0.63197, validation/bleu=26.5847, validation/loss=1.84173, validation/num_examples=3000 +I0916 09:48:51.791542 139652715497216 logging_writer.py:48] [12500] global_step=12500, grad_norm=0.20238, loss=2.10086 +I0916 09:48:51.794802 139681154073792 submission.py:307] 12500) loss = 2.101, grad_norm = 0.202 +I0916 09:50:10.616155 139652707104512 logging_writer.py:48] [13000] global_step=13000, grad_norm=0.202958, loss=2.10718 +I0916 09:50:10.619461 139681154073792 submission.py:307] 13000) loss = 2.107, grad_norm = 0.203 +I0916 09:51:29.623435 139652715497216 logging_writer.py:48] [13500] global_step=13500, grad_norm=0.20258, loss=2.05852 +I0916 09:51:29.626853 139681154073792 submission.py:307] 13500) loss = 2.059, grad_norm = 0.203 +I0916 09:52:48.631148 139652707104512 logging_writer.py:48] [14000] global_step=14000, grad_norm=0.193351, loss=2.08772 +I0916 09:52:48.634455 139681154073792 submission.py:307] 14000) loss = 2.088, grad_norm = 0.193 +I0916 09:54:07.675690 139652715497216 logging_writer.py:48] [14500] global_step=14500, grad_norm=0.184986, loss=2.07661 +I0916 09:54:07.678878 139681154073792 submission.py:307] 14500) loss = 2.077, grad_norm = 0.185 +I0916 09:55:26.722233 139652707104512 logging_writer.py:48] [15000] global_step=15000, grad_norm=0.186757, loss=2.01957 +I0916 09:55:26.725886 139681154073792 submission.py:307] 15000) loss = 2.020, grad_norm = 0.187 +I0916 09:56:45.838311 139652715497216 logging_writer.py:48] [15500] global_step=15500, grad_norm=0.179196, loss=1.95816 +I0916 09:56:45.841749 139681154073792 submission.py:307] 15500) loss = 1.958, grad_norm = 0.179 +I0916 09:58:04.885192 139652707104512 logging_writer.py:48] [16000] global_step=16000, grad_norm=0.173792, loss=2.03723 +I0916 09:58:04.888393 139681154073792 submission.py:307] 16000) loss = 2.037, grad_norm = 0.174 +I0916 09:58:47.251172 139681154073792 spec.py:333] Evaluating on the training split. +I0916 09:58:49.484363 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 09:59:48.706632 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 09:59:50.922366 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 10:00:47.078571 139681154073792 spec.py:363] Evaluating on the test split. +I0916 10:00:49.290054 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 10:01:43.406729 139681154073792 submission_runner.py:516] Time since start: 3852.87s, Step: 16265, {'train/accuracy': 0.6276181755751402, 'train/loss': 1.8615210455533937, 'train/bleu': 30.31384127236931, 'validation/accuracy': 0.644926907291912, 'validation/loss': 1.736397603563502, 'validation/bleu': 27.520962113950997, 'validation/num_examples': 3000, 'test/accuracy': 0.652431584451804, 'test/loss': 1.6740173435593515, 'test/bleu': 26.223018677187245, 'test/num_examples': 3003, 'score': 2621.048262119293, 'total_duration': 3852.870769262314, 'accumulated_submission_time': 2621.048262119293, 'accumulated_eval_time': 1220.5474019050598, 'accumulated_logging_time': 0.13915300369262695} +I0916 10:01:43.432796 139652715497216 logging_writer.py:48] [16265] accumulated_eval_time=1220.55, accumulated_logging_time=0.139153, accumulated_submission_time=2621.05, global_step=16265, preemption_count=0, score=2621.05, test/accuracy=0.652432, test/bleu=26.223, test/loss=1.67402, test/num_examples=3003, total_duration=3852.87, train/accuracy=0.627618, train/bleu=30.3138, train/loss=1.86152, validation/accuracy=0.644927, validation/bleu=27.521, validation/loss=1.7364, validation/num_examples=3000 +I0916 10:02:21.236096 139652707104512 logging_writer.py:48] [16500] global_step=16500, grad_norm=0.169286, loss=1.9695 +I0916 10:02:21.239257 139681154073792 submission.py:307] 16500) loss = 1.969, grad_norm = 0.169 +I0916 10:03:40.134922 139652715497216 logging_writer.py:48] [17000] global_step=17000, grad_norm=0.16856, loss=1.98507 +I0916 10:03:40.138347 139681154073792 submission.py:307] 17000) loss = 1.985, grad_norm = 0.169 +I0916 10:04:59.187322 139652707104512 logging_writer.py:48] [17500] global_step=17500, grad_norm=0.171518, loss=1.92703 +I0916 10:04:59.190596 139681154073792 submission.py:307] 17500) loss = 1.927, grad_norm = 0.172 +I0916 10:06:18.272267 139652715497216 logging_writer.py:48] [18000] global_step=18000, grad_norm=0.172908, loss=1.83308 +I0916 10:06:18.275639 139681154073792 submission.py:307] 18000) loss = 1.833, grad_norm = 0.173 +I0916 10:07:37.363369 139652707104512 logging_writer.py:48] [18500] global_step=18500, grad_norm=0.157748, loss=1.95023 +I0916 10:07:37.366848 139681154073792 submission.py:307] 18500) loss = 1.950, grad_norm = 0.158 +I0916 10:08:56.441594 139652715497216 logging_writer.py:48] [19000] global_step=19000, grad_norm=0.158806, loss=1.84045 +I0916 10:08:56.445152 139681154073792 submission.py:307] 19000) loss = 1.840, grad_norm = 0.159 +I0916 10:10:15.538724 139652707104512 logging_writer.py:48] [19500] global_step=19500, grad_norm=0.166165, loss=1.94764 +I0916 10:10:15.541943 139681154073792 submission.py:307] 19500) loss = 1.948, grad_norm = 0.166 +I0916 10:11:34.633173 139652715497216 logging_writer.py:48] [20000] global_step=20000, grad_norm=0.162747, loss=1.89405 +I0916 10:11:34.636404 139681154073792 submission.py:307] 20000) loss = 1.894, grad_norm = 0.163 +I0916 10:12:28.056103 139681154073792 spec.py:333] Evaluating on the training split. +I0916 10:12:30.291008 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 10:13:35.096500 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 10:13:37.320649 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 10:14:40.766332 139681154073792 spec.py:363] Evaluating on the test split. +I0916 10:14:42.985877 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 10:15:42.270356 139681154073792 submission_runner.py:516] Time since start: 4691.73s, Step: 20335, {'train/accuracy': 0.6405194984153232, 'train/loss': 1.7526106259473613, 'train/bleu': 31.23142393247607, 'validation/accuracy': 0.6531599112224274, 'validation/loss': 1.6666036378966163, 'validation/bleu': 27.66911449034538, 'validation/num_examples': 3000, 'test/accuracy': 0.6614142118412643, 'test/loss': 1.6036165097902504, 'test/bleu': 26.956141842912437, 'test/num_examples': 3003, 'score': 3263.100668668747, 'total_duration': 4691.734410762787, 'accumulated_submission_time': 3263.100668668747, 'accumulated_eval_time': 1414.761714220047, 'accumulated_logging_time': 0.1748809814453125} +I0916 10:15:42.301046 139652707104512 logging_writer.py:48] [20335] accumulated_eval_time=1414.76, accumulated_logging_time=0.174881, accumulated_submission_time=3263.1, global_step=20335, preemption_count=0, score=3263.1, test/accuracy=0.661414, test/bleu=26.9561, test/loss=1.60362, test/num_examples=3003, total_duration=4691.73, train/accuracy=0.640519, train/bleu=31.2314, train/loss=1.75261, validation/accuracy=0.65316, validation/bleu=27.6691, validation/loss=1.6666, validation/num_examples=3000 +I0916 10:16:09.068501 139652715497216 logging_writer.py:48] [20500] global_step=20500, grad_norm=0.168098, loss=1.93485 +I0916 10:16:09.071891 139681154073792 submission.py:307] 20500) loss = 1.935, grad_norm = 0.168 +I0916 10:17:27.839537 139652707104512 logging_writer.py:48] [21000] global_step=21000, grad_norm=0.153064, loss=1.84902 +I0916 10:17:27.842707 139681154073792 submission.py:307] 21000) loss = 1.849, grad_norm = 0.153 +I0916 10:18:46.879088 139652715497216 logging_writer.py:48] [21500] global_step=21500, grad_norm=0.158226, loss=1.8189 +I0916 10:18:46.882538 139681154073792 submission.py:307] 21500) loss = 1.819, grad_norm = 0.158 +I0916 10:20:05.905156 139652707104512 logging_writer.py:48] [22000] global_step=22000, grad_norm=0.170159, loss=1.90131 +I0916 10:20:05.908511 139681154073792 submission.py:307] 22000) loss = 1.901, grad_norm = 0.170 +I0916 10:21:25.016393 139652715497216 logging_writer.py:48] [22500] global_step=22500, grad_norm=0.173217, loss=1.86338 +I0916 10:21:25.019853 139681154073792 submission.py:307] 22500) loss = 1.863, grad_norm = 0.173 +I0916 10:22:44.086574 139652707104512 logging_writer.py:48] [23000] global_step=23000, grad_norm=0.14821, loss=1.95769 +I0916 10:22:44.089820 139681154073792 submission.py:307] 23000) loss = 1.958, grad_norm = 0.148 +I0916 10:24:03.181613 139652715497216 logging_writer.py:48] [23500] global_step=23500, grad_norm=0.157389, loss=1.83297 +I0916 10:24:03.185097 139681154073792 submission.py:307] 23500) loss = 1.833, grad_norm = 0.157 +I0916 10:25:22.275077 139652707104512 logging_writer.py:48] [24000] global_step=24000, grad_norm=0.157464, loss=1.83155 +I0916 10:25:22.278423 139681154073792 submission.py:307] 24000) loss = 1.832, grad_norm = 0.157 +I0916 10:26:27.000531 139681154073792 spec.py:333] Evaluating on the training split. +I0916 10:26:29.232368 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 10:27:38.490176 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 10:27:40.708503 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 10:28:43.369716 139681154073792 spec.py:363] Evaluating on the test split. +I0916 10:28:45.587388 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 10:29:39.473970 139681154073792 submission_runner.py:516] Time since start: 5528.94s, Step: 24407, {'train/accuracy': 0.6414288155870791, 'train/loss': 1.7548805759699198, 'train/bleu': 31.41940241037089, 'validation/accuracy': 0.6600042156947837, 'validation/loss': 1.6223898920968123, 'validation/bleu': 28.151750697478555, 'validation/num_examples': 3000, 'test/accuracy': 0.6718029167392946, 'test/loss': 1.5446280721631516, 'test/bleu': 27.845316861934634, 'test/num_examples': 3003, 'score': 3905.2253880500793, 'total_duration': 5528.938032865524, 'accumulated_submission_time': 3905.2253880500793, 'accumulated_eval_time': 1607.2352409362793, 'accumulated_logging_time': 0.21613717079162598} +I0916 10:29:39.500675 139652715497216 logging_writer.py:48] [24407] accumulated_eval_time=1607.24, accumulated_logging_time=0.216137, accumulated_submission_time=3905.23, global_step=24407, preemption_count=0, score=3905.23, test/accuracy=0.671803, test/bleu=27.8453, test/loss=1.54463, test/num_examples=3003, total_duration=5528.94, train/accuracy=0.641429, train/bleu=31.4194, train/loss=1.75488, validation/accuracy=0.660004, validation/bleu=28.1518, validation/loss=1.62239, validation/num_examples=3000 +I0916 10:29:54.927273 139652707104512 logging_writer.py:48] [24500] global_step=24500, grad_norm=0.203336, loss=1.81669 +I0916 10:29:54.930367 139681154073792 submission.py:307] 24500) loss = 1.817, grad_norm = 0.203 +I0916 10:31:13.658041 139652715497216 logging_writer.py:48] [25000] global_step=25000, grad_norm=0.151218, loss=1.86499 +I0916 10:31:13.661698 139681154073792 submission.py:307] 25000) loss = 1.865, grad_norm = 0.151 +I0916 10:32:32.658319 139652707104512 logging_writer.py:48] [25500] global_step=25500, grad_norm=0.208889, loss=1.79844 +I0916 10:32:32.661817 139681154073792 submission.py:307] 25500) loss = 1.798, grad_norm = 0.209 +I0916 10:33:51.723490 139652715497216 logging_writer.py:48] [26000] global_step=26000, grad_norm=0.144517, loss=1.73451 +I0916 10:33:51.726949 139681154073792 submission.py:307] 26000) loss = 1.735, grad_norm = 0.145 +I0916 10:35:10.861618 139652707104512 logging_writer.py:48] [26500] global_step=26500, grad_norm=0.154119, loss=1.83968 +I0916 10:35:10.864864 139681154073792 submission.py:307] 26500) loss = 1.840, grad_norm = 0.154 +I0916 10:36:29.929833 139652715497216 logging_writer.py:48] [27000] global_step=27000, grad_norm=0.155045, loss=1.75638 +I0916 10:36:29.933277 139681154073792 submission.py:307] 27000) loss = 1.756, grad_norm = 0.155 +I0916 10:37:49.012657 139652707104512 logging_writer.py:48] [27500] global_step=27500, grad_norm=0.157184, loss=1.85264 +I0916 10:37:49.015971 139681154073792 submission.py:307] 27500) loss = 1.853, grad_norm = 0.157 +I0916 10:39:08.127829 139652715497216 logging_writer.py:48] [28000] global_step=28000, grad_norm=0.154757, loss=1.85043 +I0916 10:39:08.131286 139681154073792 submission.py:307] 28000) loss = 1.850, grad_norm = 0.155 +I0916 10:40:24.086833 139681154073792 spec.py:333] Evaluating on the training split. +I0916 10:40:26.320422 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 10:41:49.590197 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 10:41:51.808041 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 10:43:03.655624 139681154073792 spec.py:363] Evaluating on the test split. +I0916 10:43:05.870805 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 10:44:26.698305 139681154073792 submission_runner.py:516] Time since start: 6416.16s, Step: 28478, {'train/accuracy': 0.646569454394195, 'train/loss': 1.696688394033231, 'train/bleu': 31.908780533204727, 'validation/accuracy': 0.6634015697263518, 'validation/loss': 1.5865189985245067, 'validation/bleu': 28.6387571484488, 'validation/num_examples': 3000, 'test/accuracy': 0.6753123002730812, 'test/loss': 1.5034113357736332, 'test/bleu': 28.228522569674404, 'test/num_examples': 3003, 'score': 4547.240791797638, 'total_duration': 6416.162347793579, 'accumulated_submission_time': 4547.240791797638, 'accumulated_eval_time': 1849.8468225002289, 'accumulated_logging_time': 0.25238633155822754} +I0916 10:44:26.725225 139652707104512 logging_writer.py:48] [28478] accumulated_eval_time=1849.85, accumulated_logging_time=0.252386, accumulated_submission_time=4547.24, global_step=28478, preemption_count=0, score=4547.24, test/accuracy=0.675312, test/bleu=28.2285, test/loss=1.50341, test/num_examples=3003, total_duration=6416.16, train/accuracy=0.646569, train/bleu=31.9088, train/loss=1.69669, validation/accuracy=0.663402, validation/bleu=28.6388, validation/loss=1.58652, validation/num_examples=3000 +I0916 10:44:30.972703 139652715497216 logging_writer.py:48] [28500] global_step=28500, grad_norm=0.177907, loss=1.88203 +I0916 10:44:30.975763 139681154073792 submission.py:307] 28500) loss = 1.882, grad_norm = 0.178 +I0916 10:45:49.625336 139652707104512 logging_writer.py:48] [29000] global_step=29000, grad_norm=0.154326, loss=1.81868 +I0916 10:45:49.628985 139681154073792 submission.py:307] 29000) loss = 1.819, grad_norm = 0.154 +I0916 10:47:08.556386 139652715497216 logging_writer.py:48] [29500] global_step=29500, grad_norm=0.179077, loss=1.79216 +I0916 10:47:08.559653 139681154073792 submission.py:307] 29500) loss = 1.792, grad_norm = 0.179 +I0916 10:48:27.601018 139652707104512 logging_writer.py:48] [30000] global_step=30000, grad_norm=0.150964, loss=1.79646 +I0916 10:48:27.604264 139681154073792 submission.py:307] 30000) loss = 1.796, grad_norm = 0.151 +I0916 10:49:46.667259 139652715497216 logging_writer.py:48] [30500] global_step=30500, grad_norm=0.162882, loss=1.81994 +I0916 10:49:46.670644 139681154073792 submission.py:307] 30500) loss = 1.820, grad_norm = 0.163 +I0916 10:51:05.756323 139652707104512 logging_writer.py:48] [31000] global_step=31000, grad_norm=0.156448, loss=1.73794 +I0916 10:51:05.759700 139681154073792 submission.py:307] 31000) loss = 1.738, grad_norm = 0.156 +I0916 10:52:24.813964 139652715497216 logging_writer.py:48] [31500] global_step=31500, grad_norm=0.161722, loss=1.76904 +I0916 10:52:24.817340 139681154073792 submission.py:307] 31500) loss = 1.769, grad_norm = 0.162 +I0916 10:53:43.882419 139652707104512 logging_writer.py:48] [32000] global_step=32000, grad_norm=0.163349, loss=1.76512 +I0916 10:53:43.885656 139681154073792 submission.py:307] 32000) loss = 1.765, grad_norm = 0.163 +I0916 10:55:02.888323 139652715497216 logging_writer.py:48] [32500] global_step=32500, grad_norm=0.178167, loss=1.7225 +I0916 10:55:02.891843 139681154073792 submission.py:307] 32500) loss = 1.722, grad_norm = 0.178 +I0916 10:55:11.379295 139681154073792 spec.py:333] Evaluating on the training split. +I0916 10:55:13.610345 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 10:56:33.804061 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 10:56:36.016294 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 10:57:37.002150 139681154073792 spec.py:363] Evaluating on the test split. +I0916 10:57:39.218770 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 10:58:34.944759 139681154073792 submission_runner.py:516] Time since start: 7264.41s, Step: 32551, {'train/accuracy': 0.6574741352642615, 'train/loss': 1.6197046035219578, 'train/bleu': 32.82359983605892, 'validation/accuracy': 0.6672576905432047, 'validation/loss': 1.5561068097419746, 'validation/bleu': 29.33604280909626, 'validation/num_examples': 3000, 'test/accuracy': 0.6806228574748707, 'test/loss': 1.476671351170763, 'test/bleu': 28.605820851693665, 'test/num_examples': 3003, 'score': 5189.318318843842, 'total_duration': 7264.40878868103, 'accumulated_submission_time': 5189.318318843842, 'accumulated_eval_time': 2053.412314414978, 'accumulated_logging_time': 0.2888314723968506} +I0916 10:58:34.972104 139652707104512 logging_writer.py:48] [32551] accumulated_eval_time=2053.41, accumulated_logging_time=0.288831, accumulated_submission_time=5189.32, global_step=32551, preemption_count=0, score=5189.32, test/accuracy=0.680623, test/bleu=28.6058, test/loss=1.47667, test/num_examples=3003, total_duration=7264.41, train/accuracy=0.657474, train/bleu=32.8236, train/loss=1.6197, validation/accuracy=0.667258, validation/bleu=29.336, validation/loss=1.55611, validation/num_examples=3000 +I0916 10:59:46.419368 139652715497216 logging_writer.py:48] [33000] global_step=33000, grad_norm=0.226988, loss=1.80598 +I0916 10:59:46.422785 139681154073792 submission.py:307] 33000) loss = 1.806, grad_norm = 0.227 +I0916 11:01:05.317900 139652707104512 logging_writer.py:48] [33500] global_step=33500, grad_norm=0.154265, loss=1.76378 +I0916 11:01:05.321449 139681154073792 submission.py:307] 33500) loss = 1.764, grad_norm = 0.154 +I0916 11:02:24.307063 139652715497216 logging_writer.py:48] [34000] global_step=34000, grad_norm=0.170631, loss=1.70249 +I0916 11:02:24.310358 139681154073792 submission.py:307] 34000) loss = 1.702, grad_norm = 0.171 +I0916 11:03:43.370240 139652707104512 logging_writer.py:48] [34500] global_step=34500, grad_norm=0.18952, loss=1.79238 +I0916 11:03:43.373998 139681154073792 submission.py:307] 34500) loss = 1.792, grad_norm = 0.190 +I0916 11:05:02.458122 139652715497216 logging_writer.py:48] [35000] global_step=35000, grad_norm=0.169946, loss=1.73496 +I0916 11:05:02.461386 139681154073792 submission.py:307] 35000) loss = 1.735, grad_norm = 0.170 +I0916 11:06:21.542382 139652707104512 logging_writer.py:48] [35500] global_step=35500, grad_norm=0.194112, loss=1.74424 +I0916 11:06:21.546092 139681154073792 submission.py:307] 35500) loss = 1.744, grad_norm = 0.194 +I0916 11:07:40.614491 139652715497216 logging_writer.py:48] [36000] global_step=36000, grad_norm=0.174816, loss=1.67969 +I0916 11:07:40.617887 139681154073792 submission.py:307] 36000) loss = 1.680, grad_norm = 0.175 +I0916 11:08:59.637928 139652707104512 logging_writer.py:48] [36500] global_step=36500, grad_norm=0.18117, loss=1.6927 +I0916 11:08:59.641442 139681154073792 submission.py:307] 36500) loss = 1.693, grad_norm = 0.181 +I0916 11:09:19.549530 139681154073792 spec.py:333] Evaluating on the training split. +I0916 11:09:21.780556 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 11:10:41.492599 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 11:10:43.702031 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 11:11:47.809810 139681154073792 spec.py:363] Evaluating on the test split. +I0916 11:11:50.027368 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 11:12:47.804152 139681154073792 submission_runner.py:516] Time since start: 8117.27s, Step: 36623, {'train/accuracy': 0.6546037249512426, 'train/loss': 1.64831480457122, 'train/bleu': 32.60188823846328, 'validation/accuracy': 0.6693903361396635, 'validation/loss': 1.538441091864949, 'validation/bleu': 29.129087562232385, 'validation/num_examples': 3000, 'test/accuracy': 0.6837952472256116, 'test/loss': 1.4489703169484631, 'test/bleu': 28.47953293764722, 'test/num_examples': 3003, 'score': 5831.318049192429, 'total_duration': 8117.268192768097, 'accumulated_submission_time': 5831.318049192429, 'accumulated_eval_time': 2261.6669857501984, 'accumulated_logging_time': 0.326277494430542} +I0916 11:12:47.831137 139652715497216 logging_writer.py:48] [36623] accumulated_eval_time=2261.67, accumulated_logging_time=0.326277, accumulated_submission_time=5831.32, global_step=36623, preemption_count=0, score=5831.32, test/accuracy=0.683795, test/bleu=28.4795, test/loss=1.44897, test/num_examples=3003, total_duration=8117.27, train/accuracy=0.654604, train/bleu=32.6019, train/loss=1.64831, validation/accuracy=0.66939, validation/bleu=29.1291, validation/loss=1.53844, validation/num_examples=3000 +I0916 11:13:47.890093 139652707104512 logging_writer.py:48] [37000] global_step=37000, grad_norm=0.240748, loss=1.66829 +I0916 11:13:47.893476 139681154073792 submission.py:307] 37000) loss = 1.668, grad_norm = 0.241 +I0916 11:15:06.696811 139652715497216 logging_writer.py:48] [37500] global_step=37500, grad_norm=0.212377, loss=1.79159 +I0916 11:15:06.700050 139681154073792 submission.py:307] 37500) loss = 1.792, grad_norm = 0.212 +I0916 11:16:25.692975 139652707104512 logging_writer.py:48] [38000] global_step=38000, grad_norm=0.175362, loss=1.75049 +I0916 11:16:25.696500 139681154073792 submission.py:307] 38000) loss = 1.750, grad_norm = 0.175 +I0916 11:17:44.751480 139652715497216 logging_writer.py:48] [38500] global_step=38500, grad_norm=0.220934, loss=1.69573 +I0916 11:17:44.754711 139681154073792 submission.py:307] 38500) loss = 1.696, grad_norm = 0.221 +I0916 11:19:03.831085 139652707104512 logging_writer.py:48] [39000] global_step=39000, grad_norm=0.182939, loss=1.6174 +I0916 11:19:03.834483 139681154073792 submission.py:307] 39000) loss = 1.617, grad_norm = 0.183 +I0916 11:20:22.920946 139652715497216 logging_writer.py:48] [39500] global_step=39500, grad_norm=0.195047, loss=1.76478 +I0916 11:20:22.924192 139681154073792 submission.py:307] 39500) loss = 1.765, grad_norm = 0.195 +I0916 11:21:42.027426 139652707104512 logging_writer.py:48] [40000] global_step=40000, grad_norm=0.243193, loss=1.69165 +I0916 11:21:42.030975 139681154073792 submission.py:307] 40000) loss = 1.692, grad_norm = 0.243 +I0916 11:23:01.108900 139652715497216 logging_writer.py:48] [40500] global_step=40500, grad_norm=0.237949, loss=1.75808 +I0916 11:23:01.112228 139681154073792 submission.py:307] 40500) loss = 1.758, grad_norm = 0.238 +I0916 11:23:32.526849 139681154073792 spec.py:333] Evaluating on the training split. +I0916 11:23:34.758867 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 11:25:16.731334 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 11:25:18.946399 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 11:26:43.446498 139681154073792 spec.py:363] Evaluating on the test split. +I0916 11:26:45.662371 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 11:28:07.155961 139681154073792 submission_runner.py:516] Time since start: 9036.62s, Step: 40696, {'train/accuracy': 0.6608371754979673, 'train/loss': 1.5949718509391537, 'train/bleu': 32.70192794968317, 'validation/accuracy': 0.6731596632403813, 'validation/loss': 1.5138081440403715, 'validation/bleu': 29.136297489173543, 'validation/num_examples': 3000, 'test/accuracy': 0.6858985532508279, 'test/loss': 1.4243017728487595, 'test/bleu': 28.860104183886676, 'test/num_examples': 3003, 'score': 6473.437443256378, 'total_duration': 9036.62000489235, 'accumulated_submission_time': 6473.437443256378, 'accumulated_eval_time': 2536.2961554527283, 'accumulated_logging_time': 0.362851619720459} +I0916 11:28:07.183719 139652707104512 logging_writer.py:48] [40696] accumulated_eval_time=2536.3, accumulated_logging_time=0.362852, accumulated_submission_time=6473.44, global_step=40696, preemption_count=0, score=6473.44, test/accuracy=0.685899, test/bleu=28.8601, test/loss=1.4243, test/num_examples=3003, total_duration=9036.62, train/accuracy=0.660837, train/bleu=32.7019, train/loss=1.59497, validation/accuracy=0.67316, validation/bleu=29.1363, validation/loss=1.51381, validation/num_examples=3000 +I0916 11:28:55.742676 139652715497216 logging_writer.py:48] [41000] global_step=41000, grad_norm=0.202404, loss=1.68936 +I0916 11:28:55.745951 139681154073792 submission.py:307] 41000) loss = 1.689, grad_norm = 0.202 +I0916 11:30:14.582404 139652707104512 logging_writer.py:48] [41500] global_step=41500, grad_norm=0.208391, loss=1.67561 +I0916 11:30:14.585708 139681154073792 submission.py:307] 41500) loss = 1.676, grad_norm = 0.208 +I0916 11:31:33.586931 139652715497216 logging_writer.py:48] [42000] global_step=42000, grad_norm=0.274094, loss=1.75671 +I0916 11:31:33.590263 139681154073792 submission.py:307] 42000) loss = 1.757, grad_norm = 0.274 +I0916 11:32:52.591874 139652707104512 logging_writer.py:48] [42500] global_step=42500, grad_norm=0.229465, loss=1.67699 +I0916 11:32:52.595344 139681154073792 submission.py:307] 42500) loss = 1.677, grad_norm = 0.229 +I0916 11:34:11.659330 139652715497216 logging_writer.py:48] [43000] global_step=43000, grad_norm=0.20954, loss=1.63177 +I0916 11:34:11.662628 139681154073792 submission.py:307] 43000) loss = 1.632, grad_norm = 0.210 +I0916 11:35:30.696197 139652707104512 logging_writer.py:48] [43500] global_step=43500, grad_norm=0.203519, loss=1.75522 +I0916 11:35:30.699372 139681154073792 submission.py:307] 43500) loss = 1.755, grad_norm = 0.204 +I0916 11:36:49.691673 139652715497216 logging_writer.py:48] [44000] global_step=44000, grad_norm=0.223498, loss=1.7518 +I0916 11:36:49.695172 139681154073792 submission.py:307] 44000) loss = 1.752, grad_norm = 0.223 +I0916 11:38:08.786529 139652707104512 logging_writer.py:48] [44500] global_step=44500, grad_norm=0.236519, loss=1.70998 +I0916 11:38:08.789841 139681154073792 submission.py:307] 44500) loss = 1.710, grad_norm = 0.237 +I0916 11:38:51.755372 139681154073792 spec.py:333] Evaluating on the training split. +I0916 11:38:53.985303 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 11:40:13.139526 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 11:40:15.350179 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 11:41:20.989037 139681154073792 spec.py:363] Evaluating on the test split. +I0916 11:41:23.206282 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 11:42:19.358209 139681154073792 submission_runner.py:516] Time since start: 9888.82s, Step: 44769, {'train/accuracy': 0.6736368540047972, 'train/loss': 1.5027317254943593, 'train/bleu': 33.463669608567024, 'validation/accuracy': 0.67566428190599, 'validation/loss': 1.4967648029782643, 'validation/bleu': 29.447153797348868, 'validation/num_examples': 3000, 'test/accuracy': 0.6869095345999652, 'test/loss': 1.413147659926791, 'test/bleu': 29.056115384466224, 'test/num_examples': 3003, 'score': 7115.437336683273, 'total_duration': 9888.822259187698, 'accumulated_submission_time': 7115.437336683273, 'accumulated_eval_time': 2743.8992309570312, 'accumulated_logging_time': 0.4001736640930176} +I0916 11:42:19.384771 139652715497216 logging_writer.py:48] [44769] accumulated_eval_time=2743.9, accumulated_logging_time=0.400174, accumulated_submission_time=7115.44, global_step=44769, preemption_count=0, score=7115.44, test/accuracy=0.68691, test/bleu=29.0561, test/loss=1.41315, test/num_examples=3003, total_duration=9888.82, train/accuracy=0.673637, train/bleu=33.4637, train/loss=1.50273, validation/accuracy=0.675664, validation/bleu=29.4472, validation/loss=1.49676, validation/num_examples=3000 +I0916 11:42:56.544557 139652707104512 logging_writer.py:48] [45000] global_step=45000, grad_norm=0.23506, loss=1.68287 +I0916 11:42:56.547839 139681154073792 submission.py:307] 45000) loss = 1.683, grad_norm = 0.235 +I0916 11:44:15.340085 139652715497216 logging_writer.py:48] [45500] global_step=45500, grad_norm=0.287116, loss=1.64769 +I0916 11:44:15.343438 139681154073792 submission.py:307] 45500) loss = 1.648, grad_norm = 0.287 +I0916 11:45:34.305658 139652707104512 logging_writer.py:48] [46000] global_step=46000, grad_norm=0.254616, loss=1.71205 +I0916 11:45:34.308852 139681154073792 submission.py:307] 46000) loss = 1.712, grad_norm = 0.255 +I0916 11:46:53.328931 139652715497216 logging_writer.py:48] [46500] global_step=46500, grad_norm=0.236796, loss=1.61907 +I0916 11:46:53.332146 139681154073792 submission.py:307] 46500) loss = 1.619, grad_norm = 0.237 +I0916 11:48:12.370882 139652707104512 logging_writer.py:48] [47000] global_step=47000, grad_norm=0.237498, loss=1.67687 +I0916 11:48:12.374408 139681154073792 submission.py:307] 47000) loss = 1.677, grad_norm = 0.237 +I0916 11:49:31.390746 139652715497216 logging_writer.py:48] [47500] global_step=47500, grad_norm=0.303485, loss=1.66961 +I0916 11:49:31.394016 139681154073792 submission.py:307] 47500) loss = 1.670, grad_norm = 0.303 +I0916 11:50:50.425933 139652707104512 logging_writer.py:48] [48000] global_step=48000, grad_norm=0.265766, loss=1.60315 +I0916 11:50:50.429223 139681154073792 submission.py:307] 48000) loss = 1.603, grad_norm = 0.266 +I0916 11:52:09.405999 139652715497216 logging_writer.py:48] [48500] global_step=48500, grad_norm=0.248976, loss=1.67526 +I0916 11:52:09.409273 139681154073792 submission.py:307] 48500) loss = 1.675, grad_norm = 0.249 +I0916 11:53:03.998476 139681154073792 spec.py:333] Evaluating on the training split. +I0916 11:53:06.230443 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 11:54:47.859642 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 11:54:50.080684 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 11:56:02.577753 139681154073792 spec.py:363] Evaluating on the test split. +I0916 11:56:04.791606 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 11:57:08.065054 139681154073792 submission_runner.py:516] Time since start: 10777.53s, Step: 48843, {'train/accuracy': 0.6602731896896611, 'train/loss': 1.5895205249471338, 'train/bleu': 33.20233996811842, 'validation/accuracy': 0.6778093266047538, 'validation/loss': 1.4871544331440403, 'validation/bleu': 29.348687414056066, 'validation/num_examples': 3000, 'test/accuracy': 0.690163267677648, 'test/loss': 1.3950370584219394, 'test/bleu': 29.252620058582394, 'test/num_examples': 3003, 'score': 7757.446277618408, 'total_duration': 10777.529091835022, 'accumulated_submission_time': 7757.446277618408, 'accumulated_eval_time': 2987.965854883194, 'accumulated_logging_time': 0.46700501441955566} +I0916 11:57:08.091934 139652707104512 logging_writer.py:48] [48843] accumulated_eval_time=2987.97, accumulated_logging_time=0.467005, accumulated_submission_time=7757.45, global_step=48843, preemption_count=0, score=7757.45, test/accuracy=0.690163, test/bleu=29.2526, test/loss=1.39504, test/num_examples=3003, total_duration=10777.5, train/accuracy=0.660273, train/bleu=33.2023, train/loss=1.58952, validation/accuracy=0.677809, validation/bleu=29.3487, validation/loss=1.48715, validation/num_examples=3000 +I0916 11:57:33.585066 139652715497216 logging_writer.py:48] [49000] global_step=49000, grad_norm=0.237456, loss=1.6038 +I0916 11:57:33.588331 139681154073792 submission.py:307] 49000) loss = 1.604, grad_norm = 0.237 +I0916 11:58:52.371894 139652707104512 logging_writer.py:48] [49500] global_step=49500, grad_norm=0.234422, loss=1.72078 +I0916 11:58:52.375148 139681154073792 submission.py:307] 49500) loss = 1.721, grad_norm = 0.234 +I0916 12:00:11.326548 139652715497216 logging_writer.py:48] [50000] global_step=50000, grad_norm=0.26852, loss=1.60806 +I0916 12:00:11.329905 139681154073792 submission.py:307] 50000) loss = 1.608, grad_norm = 0.269 +I0916 12:01:30.313688 139652707104512 logging_writer.py:48] [50500] global_step=50500, grad_norm=0.263085, loss=1.67428 +I0916 12:01:30.317108 139681154073792 submission.py:307] 50500) loss = 1.674, grad_norm = 0.263 +I0916 12:02:49.317526 139652715497216 logging_writer.py:48] [51000] global_step=51000, grad_norm=0.258746, loss=1.69353 +I0916 12:02:49.320772 139681154073792 submission.py:307] 51000) loss = 1.694, grad_norm = 0.259 +I0916 12:04:08.320321 139652707104512 logging_writer.py:48] [51500] global_step=51500, grad_norm=0.286934, loss=1.64239 +I0916 12:04:08.323587 139681154073792 submission.py:307] 51500) loss = 1.642, grad_norm = 0.287 +I0916 12:05:27.328333 139652715497216 logging_writer.py:48] [52000] global_step=52000, grad_norm=0.255317, loss=1.66099 +I0916 12:05:27.331721 139681154073792 submission.py:307] 52000) loss = 1.661, grad_norm = 0.255 +I0916 12:06:46.262074 139652707104512 logging_writer.py:48] [52500] global_step=52500, grad_norm=0.28364, loss=1.62424 +I0916 12:06:46.265521 139681154073792 submission.py:307] 52500) loss = 1.624, grad_norm = 0.284 +I0916 12:07:52.696285 139681154073792 spec.py:333] Evaluating on the training split. +I0916 12:07:54.924714 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 12:09:40.507435 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 12:09:42.727767 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 12:11:15.773724 139681154073792 spec.py:363] Evaluating on the test split. +I0916 12:11:17.985704 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 12:12:40.438657 139681154073792 submission_runner.py:516] Time since start: 11709.90s, Step: 52918, {'train/accuracy': 0.6706479784121057, 'train/loss': 1.5236382675600897, 'train/bleu': 33.46744917235679, 'validation/accuracy': 0.6784416808223085, 'validation/loss': 1.4695985635639979, 'validation/bleu': 29.528170474094818, 'validation/num_examples': 3000, 'test/accuracy': 0.6934518621811632, 'test/loss': 1.3803054732438558, 'test/bleu': 29.445593184963908, 'test/num_examples': 3003, 'score': 8399.471651315689, 'total_duration': 11709.902684926987, 'accumulated_submission_time': 8399.471651315689, 'accumulated_eval_time': 3275.7083344459534, 'accumulated_logging_time': 0.5038421154022217} +I0916 12:12:40.467239 139652715497216 logging_writer.py:48] [52918] accumulated_eval_time=3275.71, accumulated_logging_time=0.503842, accumulated_submission_time=8399.47, global_step=52918, preemption_count=0, score=8399.47, test/accuracy=0.693452, test/bleu=29.4456, test/loss=1.38031, test/num_examples=3003, total_duration=11709.9, train/accuracy=0.670648, train/bleu=33.4674, train/loss=1.52364, validation/accuracy=0.678442, validation/bleu=29.5282, validation/loss=1.4696, validation/num_examples=3000 +I0916 12:12:54.125332 139652707104512 logging_writer.py:48] [53000] global_step=53000, grad_norm=0.308653, loss=1.52866 +I0916 12:12:54.128379 139681154073792 submission.py:307] 53000) loss = 1.529, grad_norm = 0.309 +I0916 12:14:12.769334 139652715497216 logging_writer.py:48] [53500] global_step=53500, grad_norm=0.32873, loss=1.66942 +I0916 12:14:12.772513 139681154073792 submission.py:307] 53500) loss = 1.669, grad_norm = 0.329 +I0916 12:15:31.607584 139652707104512 logging_writer.py:48] [54000] global_step=54000, grad_norm=0.272642, loss=1.69686 +I0916 12:15:31.610986 139681154073792 submission.py:307] 54000) loss = 1.697, grad_norm = 0.273 +I0916 12:16:50.519110 139652715497216 logging_writer.py:48] [54500] global_step=54500, grad_norm=0.306392, loss=1.62574 +I0916 12:16:50.522648 139681154073792 submission.py:307] 54500) loss = 1.626, grad_norm = 0.306 +I0916 12:18:09.478193 139652707104512 logging_writer.py:48] [55000] global_step=55000, grad_norm=0.31262, loss=1.60681 +I0916 12:18:09.481562 139681154073792 submission.py:307] 55000) loss = 1.607, grad_norm = 0.313 +I0916 12:19:28.456467 139652715497216 logging_writer.py:48] [55500] global_step=55500, grad_norm=0.302463, loss=1.61427 +I0916 12:19:28.459796 139681154073792 submission.py:307] 55500) loss = 1.614, grad_norm = 0.302 +I0916 12:20:47.494765 139652707104512 logging_writer.py:48] [56000] global_step=56000, grad_norm=0.264041, loss=1.64762 +I0916 12:20:47.498105 139681154073792 submission.py:307] 56000) loss = 1.648, grad_norm = 0.264 +I0916 12:22:06.477156 139652715497216 logging_writer.py:48] [56500] global_step=56500, grad_norm=0.319982, loss=1.59237 +I0916 12:22:06.480662 139681154073792 submission.py:307] 56500) loss = 1.592, grad_norm = 0.320 +I0916 12:23:25.141855 139681154073792 spec.py:333] Evaluating on the training split. +I0916 12:23:27.371881 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 12:25:03.564828 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 12:25:05.779948 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 12:26:11.921297 139681154073792 spec.py:363] Evaluating on the test split. +I0916 12:26:14.133088 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 12:27:20.981905 139681154073792 submission_runner.py:516] Time since start: 12590.45s, Step: 56995, {'train/accuracy': 0.6831654775287344, 'train/loss': 1.438460678152745, 'train/bleu': 34.52225552751289, 'validation/accuracy': 0.6807355147487322, 'validation/loss': 1.462125585857584, 'validation/bleu': 29.87392002459043, 'validation/num_examples': 3000, 'test/accuracy': 0.6953343791761083, 'test/loss': 1.3676087422288072, 'test/bleu': 29.502131903352502, 'test/num_examples': 3003, 'score': 9041.56005358696, 'total_duration': 12590.445948839188, 'accumulated_submission_time': 9041.56005358696, 'accumulated_eval_time': 3511.5483932495117, 'accumulated_logging_time': 0.5425922870635986} +I0916 12:27:21.010025 139652707104512 logging_writer.py:48] [56995] accumulated_eval_time=3511.55, accumulated_logging_time=0.542592, accumulated_submission_time=9041.56, global_step=56995, preemption_count=0, score=9041.56, test/accuracy=0.695334, test/bleu=29.5021, test/loss=1.36761, test/num_examples=3003, total_duration=12590.4, train/accuracy=0.683165, train/bleu=34.5223, train/loss=1.43846, validation/accuracy=0.680736, validation/bleu=29.8739, validation/loss=1.46213, validation/num_examples=3000 +I0916 12:27:22.593361 139652715497216 logging_writer.py:48] [57000] global_step=57000, grad_norm=0.306328, loss=1.58563 +I0916 12:27:22.596412 139681154073792 submission.py:307] 57000) loss = 1.586, grad_norm = 0.306 +I0916 12:28:41.168586 139652707104512 logging_writer.py:48] [57500] global_step=57500, grad_norm=0.397821, loss=1.67266 +I0916 12:28:41.171863 139681154073792 submission.py:307] 57500) loss = 1.673, grad_norm = 0.398 +I0916 12:30:00.021741 139652715497216 logging_writer.py:48] [58000] global_step=58000, grad_norm=0.299034, loss=1.56027 +I0916 12:30:00.025184 139681154073792 submission.py:307] 58000) loss = 1.560, grad_norm = 0.299 +I0916 12:31:18.934125 139652707104512 logging_writer.py:48] [58500] global_step=58500, grad_norm=0.290646, loss=1.61762 +I0916 12:31:18.937577 139681154073792 submission.py:307] 58500) loss = 1.618, grad_norm = 0.291 +I0916 12:32:37.871697 139652715497216 logging_writer.py:48] [59000] global_step=59000, grad_norm=0.283322, loss=1.58117 +I0916 12:32:37.875039 139681154073792 submission.py:307] 59000) loss = 1.581, grad_norm = 0.283 +I0916 12:33:56.824319 139652707104512 logging_writer.py:48] [59500] global_step=59500, grad_norm=0.31702, loss=1.61808 +I0916 12:33:56.827584 139681154073792 submission.py:307] 59500) loss = 1.618, grad_norm = 0.317 +I0916 12:35:15.782226 139652715497216 logging_writer.py:48] [60000] global_step=60000, grad_norm=0.307494, loss=1.60156 +I0916 12:35:15.785852 139681154073792 submission.py:307] 60000) loss = 1.602, grad_norm = 0.307 +I0916 12:36:34.750241 139652707104512 logging_writer.py:48] [60500] global_step=60500, grad_norm=0.309558, loss=1.53001 +I0916 12:36:34.753519 139681154073792 submission.py:307] 60500) loss = 1.530, grad_norm = 0.310 +I0916 12:37:53.751217 139652715497216 logging_writer.py:48] [61000] global_step=61000, grad_norm=0.306699, loss=1.6037 +I0916 12:37:53.754519 139681154073792 submission.py:307] 61000) loss = 1.604, grad_norm = 0.307 +I0916 12:38:05.691341 139681154073792 spec.py:333] Evaluating on the training split. +I0916 12:38:07.924462 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 12:40:06.114523 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 12:40:08.328549 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 12:41:44.509582 139681154073792 spec.py:363] Evaluating on the test split. +I0916 12:41:46.718619 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 12:43:26.870007 139681154073792 submission_runner.py:516] Time since start: 13556.33s, Step: 61073, {'train/accuracy': 0.6728275450886868, 'train/loss': 1.5079797373218524, 'train/bleu': 33.58934435863579, 'validation/accuracy': 0.6837484966088455, 'validation/loss': 1.4534882549503416, 'validation/bleu': 29.378392460263147, 'validation/num_examples': 3000, 'test/accuracy': 0.6959967462669223, 'test/loss': 1.3575914386148393, 'test/bleu': 29.822738987449174, 'test/num_examples': 3003, 'score': 9683.668237447739, 'total_duration': 13556.334055900574, 'accumulated_submission_time': 9683.668237447739, 'accumulated_eval_time': 3832.7271070480347, 'accumulated_logging_time': 0.5803112983703613} +I0916 12:43:26.898030 139652707104512 logging_writer.py:48] [61073] accumulated_eval_time=3832.73, accumulated_logging_time=0.580311, accumulated_submission_time=9683.67, global_step=61073, preemption_count=0, score=9683.67, test/accuracy=0.695997, test/bleu=29.8227, test/loss=1.35759, test/num_examples=3003, total_duration=13556.3, train/accuracy=0.672828, train/bleu=33.5893, train/loss=1.50798, validation/accuracy=0.683748, validation/bleu=29.3784, validation/loss=1.45349, validation/num_examples=3000 +I0916 12:44:34.871845 139652715497216 logging_writer.py:48] [61500] global_step=61500, grad_norm=0.311143, loss=1.57915 +I0916 12:44:34.875194 139681154073792 submission.py:307] 61500) loss = 1.579, grad_norm = 0.311 +I0916 12:45:53.788833 139652707104512 logging_writer.py:48] [62000] global_step=62000, grad_norm=0.344483, loss=1.68881 +I0916 12:45:53.792038 139681154073792 submission.py:307] 62000) loss = 1.689, grad_norm = 0.344 +I0916 12:47:12.722836 139652715497216 logging_writer.py:48] [62500] global_step=62500, grad_norm=0.303285, loss=1.63058 +I0916 12:47:12.726237 139681154073792 submission.py:307] 62500) loss = 1.631, grad_norm = 0.303 +I0916 12:48:31.746974 139652707104512 logging_writer.py:48] [63000] global_step=63000, grad_norm=0.329848, loss=1.67109 +I0916 12:48:31.750498 139681154073792 submission.py:307] 63000) loss = 1.671, grad_norm = 0.330 +I0916 12:49:50.777426 139652715497216 logging_writer.py:48] [63500] global_step=63500, grad_norm=0.312691, loss=1.61126 +I0916 12:49:50.780791 139681154073792 submission.py:307] 63500) loss = 1.611, grad_norm = 0.313 +I0916 12:51:09.829675 139652707104512 logging_writer.py:48] [64000] global_step=64000, grad_norm=0.337966, loss=1.64192 +I0916 12:51:09.832991 139681154073792 submission.py:307] 64000) loss = 1.642, grad_norm = 0.338 +I0916 12:52:28.912901 139652715497216 logging_writer.py:48] [64500] global_step=64500, grad_norm=0.328546, loss=1.62614 +I0916 12:52:28.916362 139681154073792 submission.py:307] 64500) loss = 1.626, grad_norm = 0.329 +I0916 12:53:47.991726 139652707104512 logging_writer.py:48] [65000] global_step=65000, grad_norm=0.303784, loss=1.59258 +I0916 12:53:47.995160 139681154073792 submission.py:307] 65000) loss = 1.593, grad_norm = 0.304 +I0916 12:54:11.479659 139681154073792 spec.py:333] Evaluating on the training split. +I0916 12:54:13.712759 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 12:56:07.209119 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 12:56:09.419039 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 12:57:58.612276 139681154073792 spec.py:363] Evaluating on the test split. +I0916 12:58:00.830336 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 13:00:03.844214 139681154073792 submission_runner.py:516] Time since start: 14553.31s, Step: 65146, {'train/accuracy': 0.6787732503603376, 'train/loss': 1.4658047398705072, 'train/bleu': 34.23505056955816, 'validation/accuracy': 0.6824837881737362, 'validation/loss': 1.4427903482287883, 'validation/bleu': 29.899171017447795, 'validation/num_examples': 3000, 'test/accuracy': 0.6989599674626692, 'test/loss': 1.3438375167044332, 'test/bleu': 29.895968340980154, 'test/num_examples': 3003, 'score': 10325.675132989883, 'total_duration': 14553.30824804306, 'accumulated_submission_time': 10325.675132989883, 'accumulated_eval_time': 4185.091699361801, 'accumulated_logging_time': 0.6182959079742432} +I0916 13:00:03.873854 139652715497216 logging_writer.py:48] [65146] accumulated_eval_time=4185.09, accumulated_logging_time=0.618296, accumulated_submission_time=10325.7, global_step=65146, preemption_count=0, score=10325.7, test/accuracy=0.69896, test/bleu=29.896, test/loss=1.34384, test/num_examples=3003, total_duration=14553.3, train/accuracy=0.678773, train/bleu=34.2351, train/loss=1.4658, validation/accuracy=0.682484, validation/bleu=29.8992, validation/loss=1.44279, validation/num_examples=3000 +I0916 13:01:00.392928 139652707104512 logging_writer.py:48] [65500] global_step=65500, grad_norm=0.296476, loss=1.58291 +I0916 13:01:00.396323 139681154073792 submission.py:307] 65500) loss = 1.583, grad_norm = 0.296 +I0916 13:02:19.271100 139652715497216 logging_writer.py:48] [66000] global_step=66000, grad_norm=0.311355, loss=1.59337 +I0916 13:02:19.274419 139681154073792 submission.py:307] 66000) loss = 1.593, grad_norm = 0.311 +I0916 13:03:38.248199 139652707104512 logging_writer.py:48] [66500] global_step=66500, grad_norm=0.367677, loss=1.64004 +I0916 13:03:38.251458 139681154073792 submission.py:307] 66500) loss = 1.640, grad_norm = 0.368 +I0916 13:04:57.222218 139652715497216 logging_writer.py:48] [67000] global_step=67000, grad_norm=0.333838, loss=1.62922 +I0916 13:04:57.225818 139681154073792 submission.py:307] 67000) loss = 1.629, grad_norm = 0.334 +I0916 13:06:16.258284 139652707104512 logging_writer.py:48] [67500] global_step=67500, grad_norm=0.318461, loss=1.54688 +I0916 13:06:16.261678 139681154073792 submission.py:307] 67500) loss = 1.547, grad_norm = 0.318 +I0916 13:07:35.342434 139652715497216 logging_writer.py:48] [68000] global_step=68000, grad_norm=0.314456, loss=1.47938 +I0916 13:07:35.345658 139681154073792 submission.py:307] 68000) loss = 1.479, grad_norm = 0.314 +I0916 13:08:54.382838 139652707104512 logging_writer.py:48] [68500] global_step=68500, grad_norm=0.324209, loss=1.61147 +I0916 13:08:54.386223 139681154073792 submission.py:307] 68500) loss = 1.611, grad_norm = 0.324 +I0916 13:10:13.365609 139652715497216 logging_writer.py:48] [69000] global_step=69000, grad_norm=0.328982, loss=1.59667 +I0916 13:10:13.369298 139681154073792 submission.py:307] 69000) loss = 1.597, grad_norm = 0.329 +I0916 13:10:48.517749 139681154073792 spec.py:333] Evaluating on the training split. +I0916 13:10:50.746713 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 13:12:38.614001 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 13:12:40.832878 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 13:14:20.680010 139681154073792 spec.py:363] Evaluating on the test split. +I0916 13:14:22.893334 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 13:16:01.856954 139681154073792 submission_runner.py:516] Time since start: 15511.32s, Step: 69220, {'train/accuracy': 0.6870573523560507, 'train/loss': 1.4177981832262831, 'train/bleu': 34.462200462819816, 'validation/accuracy': 0.6847900212024649, 'validation/loss': 1.4348646095522684, 'validation/bleu': 28.746716660696627, 'validation/num_examples': 3000, 'test/accuracy': 0.6989134855615595, 'test/loss': 1.337881478414967, 'test/bleu': 29.556654819001654, 'test/num_examples': 3003, 'score': 10967.742772340775, 'total_duration': 15511.32098197937, 'accumulated_submission_time': 10967.742772340775, 'accumulated_eval_time': 4498.430937290192, 'accumulated_logging_time': 0.6577272415161133} +I0916 13:16:01.886747 139652707104512 logging_writer.py:48] [69220] accumulated_eval_time=4498.43, accumulated_logging_time=0.657727, accumulated_submission_time=10967.7, global_step=69220, preemption_count=0, score=10967.7, test/accuracy=0.698913, test/bleu=29.5567, test/loss=1.33788, test/num_examples=3003, total_duration=15511.3, train/accuracy=0.687057, train/bleu=34.4622, train/loss=1.4178, validation/accuracy=0.68479, validation/bleu=28.7467, validation/loss=1.43486, validation/num_examples=3000 +I0916 13:16:46.616667 139652715497216 logging_writer.py:48] [69500] global_step=69500, grad_norm=0.34553, loss=1.61314 +I0916 13:16:46.620094 139681154073792 submission.py:307] 69500) loss = 1.613, grad_norm = 0.346 +I0916 13:18:05.424034 139652707104512 logging_writer.py:48] [70000] global_step=70000, grad_norm=0.328616, loss=1.56002 +I0916 13:18:05.427347 139681154073792 submission.py:307] 70000) loss = 1.560, grad_norm = 0.329 +I0916 13:19:24.295518 139652715497216 logging_writer.py:48] [70500] global_step=70500, grad_norm=0.338459, loss=1.56 +I0916 13:19:24.298916 139681154073792 submission.py:307] 70500) loss = 1.560, grad_norm = 0.338 +I0916 13:20:43.246875 139652707104512 logging_writer.py:48] [71000] global_step=71000, grad_norm=0.362239, loss=1.55596 +I0916 13:20:43.250343 139681154073792 submission.py:307] 71000) loss = 1.556, grad_norm = 0.362 +I0916 13:22:02.203992 139652715497216 logging_writer.py:48] [71500] global_step=71500, grad_norm=0.303191, loss=1.52941 +I0916 13:22:02.207347 139681154073792 submission.py:307] 71500) loss = 1.529, grad_norm = 0.303 +I0916 13:23:21.025101 139652707104512 logging_writer.py:48] [72000] global_step=72000, grad_norm=0.336708, loss=1.48166 +I0916 13:23:21.028756 139681154073792 submission.py:307] 72000) loss = 1.482, grad_norm = 0.337 +I0916 13:24:39.850980 139652715497216 logging_writer.py:48] [72500] global_step=72500, grad_norm=0.365393, loss=1.53067 +I0916 13:24:39.854292 139681154073792 submission.py:307] 72500) loss = 1.531, grad_norm = 0.365 +I0916 13:25:58.618099 139652707104512 logging_writer.py:48] [73000] global_step=73000, grad_norm=0.346823, loss=1.56506 +I0916 13:25:58.621420 139681154073792 submission.py:307] 73000) loss = 1.565, grad_norm = 0.347 +I0916 13:26:46.504406 139681154073792 spec.py:333] Evaluating on the training split. +I0916 13:26:48.735494 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 13:28:41.240917 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 13:28:43.460455 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 13:30:35.753587 139681154073792 spec.py:363] Evaluating on the test split. +I0916 13:30:37.973698 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 13:32:26.250719 139681154073792 submission_runner.py:516] Time since start: 16495.71s, Step: 73301, {'train/accuracy': 0.6781031888779538, 'train/loss': 1.4760303899277158, 'train/bleu': 34.477809736742664, 'validation/accuracy': 0.686451500911334, 'validation/loss': 1.4277885968555877, 'validation/bleu': 28.023441115401464, 'validation/num_examples': 3000, 'test/accuracy': 0.7004822497240137, 'test/loss': 1.3271995380861077, 'test/bleu': 28.9335588426571, 'test/num_examples': 3003, 'score': 11609.769958019257, 'total_duration': 16495.714752435684, 'accumulated_submission_time': 11609.769958019257, 'accumulated_eval_time': 4838.17728638649, 'accumulated_logging_time': 0.6972668170928955} +I0916 13:32:26.281362 139652715497216 logging_writer.py:48] [73301] accumulated_eval_time=4838.18, accumulated_logging_time=0.697267, accumulated_submission_time=11609.8, global_step=73301, preemption_count=0, score=11609.8, test/accuracy=0.700482, test/bleu=28.9336, test/loss=1.3272, test/num_examples=3003, total_duration=16495.7, train/accuracy=0.678103, train/bleu=34.4778, train/loss=1.47603, validation/accuracy=0.686452, validation/bleu=28.0234, validation/loss=1.42779, validation/num_examples=3000 +I0916 13:32:58.277545 139652707104512 logging_writer.py:48] [73500] global_step=73500, grad_norm=0.294654, loss=1.52547 +I0916 13:32:58.280947 139681154073792 submission.py:307] 73500) loss = 1.525, grad_norm = 0.295 +I0916 13:34:16.836784 139652715497216 logging_writer.py:48] [74000] global_step=74000, grad_norm=0.320534, loss=1.60937 +I0916 13:34:16.840138 139681154073792 submission.py:307] 74000) loss = 1.609, grad_norm = 0.321 +I0916 13:35:35.530574 139652707104512 logging_writer.py:48] [74500] global_step=74500, grad_norm=0.383473, loss=1.57651 +I0916 13:35:35.534106 139681154073792 submission.py:307] 74500) loss = 1.577, grad_norm = 0.383 +I0916 13:36:54.227338 139652715497216 logging_writer.py:48] [75000] global_step=75000, grad_norm=0.330018, loss=1.46294 +I0916 13:36:54.231072 139681154073792 submission.py:307] 75000) loss = 1.463, grad_norm = 0.330 +I0916 13:38:13.084654 139652707104512 logging_writer.py:48] [75500] global_step=75500, grad_norm=0.349721, loss=1.50807 +I0916 13:38:13.088563 139681154073792 submission.py:307] 75500) loss = 1.508, grad_norm = 0.350 +I0916 13:39:31.925592 139652715497216 logging_writer.py:48] [76000] global_step=76000, grad_norm=0.338549, loss=1.4955 +I0916 13:39:31.928805 139681154073792 submission.py:307] 76000) loss = 1.495, grad_norm = 0.339 +I0916 13:40:50.739102 139652707104512 logging_writer.py:48] [76500] global_step=76500, grad_norm=0.357877, loss=1.49215 +I0916 13:40:50.742507 139681154073792 submission.py:307] 76500) loss = 1.492, grad_norm = 0.358 +I0916 13:42:09.587630 139652715497216 logging_writer.py:48] [77000] global_step=77000, grad_norm=0.341866, loss=1.53974 +I0916 13:42:09.590996 139681154073792 submission.py:307] 77000) loss = 1.540, grad_norm = 0.342 +I0916 13:43:10.859150 139681154073792 spec.py:333] Evaluating on the training split. +I0916 13:43:13.091453 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 13:44:51.316528 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 13:44:53.533619 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 13:46:32.971801 139681154073792 spec.py:363] Evaluating on the test split. +I0916 13:46:35.186097 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 13:48:10.604218 139681154073792 submission_runner.py:516] Time since start: 17440.07s, Step: 77386, {'train/accuracy': 0.6885287531454164, 'train/loss': 1.4049029177343768, 'train/bleu': 35.03851466280576, 'validation/accuracy': 0.6878402003694932, 'validation/loss': 1.4182358634734846, 'validation/bleu': 30.261584758257122, 'validation/num_examples': 3000, 'test/accuracy': 0.7026552786008948, 'test/loss': 1.3144139285921794, 'test/bleu': 30.07638390589513, 'test/num_examples': 3003, 'score': 12251.764683485031, 'total_duration': 17440.068254709244, 'accumulated_submission_time': 12251.764683485031, 'accumulated_eval_time': 5137.922399997711, 'accumulated_logging_time': 0.7374632358551025} +I0916 13:48:10.634161 139652707104512 logging_writer.py:48] [77386] accumulated_eval_time=5137.92, accumulated_logging_time=0.737463, accumulated_submission_time=12251.8, global_step=77386, preemption_count=0, score=12251.8, test/accuracy=0.702655, test/bleu=30.0764, test/loss=1.31441, test/num_examples=3003, total_duration=17440.1, train/accuracy=0.688529, train/bleu=35.0385, train/loss=1.4049, validation/accuracy=0.68784, validation/bleu=30.2616, validation/loss=1.41824, validation/num_examples=3000 +I0916 13:48:29.274292 139652715497216 logging_writer.py:48] [77500] global_step=77500, grad_norm=0.360935, loss=1.50195 +I0916 13:48:29.277364 139681154073792 submission.py:307] 77500) loss = 1.502, grad_norm = 0.361 +I0916 13:49:47.799021 139652707104512 logging_writer.py:48] [78000] global_step=78000, grad_norm=0.354724, loss=1.62317 +I0916 13:49:47.802362 139681154073792 submission.py:307] 78000) loss = 1.623, grad_norm = 0.355 +I0916 13:51:06.488968 139652715497216 logging_writer.py:48] [78500] global_step=78500, grad_norm=0.353475, loss=1.4987 +I0916 13:51:06.492445 139681154073792 submission.py:307] 78500) loss = 1.499, grad_norm = 0.353 +I0916 13:52:25.286338 139652707104512 logging_writer.py:48] [79000] global_step=79000, grad_norm=0.3873, loss=1.50093 +I0916 13:52:25.289764 139681154073792 submission.py:307] 79000) loss = 1.501, grad_norm = 0.387 +I0916 13:53:44.067726 139652715497216 logging_writer.py:48] [79500] global_step=79500, grad_norm=0.338095, loss=1.59004 +I0916 13:53:44.070919 139681154073792 submission.py:307] 79500) loss = 1.590, grad_norm = 0.338 +I0916 13:55:02.843233 139652707104512 logging_writer.py:48] [80000] global_step=80000, grad_norm=0.335376, loss=1.49864 +I0916 13:55:02.846735 139681154073792 submission.py:307] 80000) loss = 1.499, grad_norm = 0.335 +I0916 13:56:21.626032 139652715497216 logging_writer.py:48] [80500] global_step=80500, grad_norm=0.3409, loss=1.56607 +I0916 13:56:21.629347 139681154073792 submission.py:307] 80500) loss = 1.566, grad_norm = 0.341 +I0916 13:57:40.394989 139652707104512 logging_writer.py:48] [81000] global_step=81000, grad_norm=0.330279, loss=1.50886 +I0916 13:57:40.398343 139681154073792 submission.py:307] 81000) loss = 1.509, grad_norm = 0.330 +I0916 13:58:55.319689 139681154073792 spec.py:333] Evaluating on the training split. +I0916 13:58:57.550826 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 14:00:45.309729 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 14:00:47.519640 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 14:02:13.285575 139681154073792 spec.py:363] Evaluating on the test split. +I0916 14:02:15.500641 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 14:03:43.134977 139681154073792 submission_runner.py:516] Time since start: 18372.60s, Step: 81473, {'train/accuracy': 0.6918920154002582, 'train/loss': 1.3938208320480745, 'train/bleu': 35.61374162031927, 'validation/accuracy': 0.6870466578219737, 'validation/loss': 1.4101627885891062, 'validation/bleu': 29.677615166629305, 'validation/num_examples': 3000, 'test/accuracy': 0.7050723374586021, 'test/loss': 1.3048747893788857, 'test/bleu': 29.864489167882073, 'test/num_examples': 3003, 'score': 12893.859102725983, 'total_duration': 18372.599026203156, 'accumulated_submission_time': 12893.859102725983, 'accumulated_eval_time': 5425.737843036652, 'accumulated_logging_time': 0.7771332263946533} +I0916 14:03:43.164701 139652715497216 logging_writer.py:48] [81473] accumulated_eval_time=5425.74, accumulated_logging_time=0.777133, accumulated_submission_time=12893.9, global_step=81473, preemption_count=0, score=12893.9, test/accuracy=0.705072, test/bleu=29.8645, test/loss=1.30487, test/num_examples=3003, total_duration=18372.6, train/accuracy=0.691892, train/bleu=35.6137, train/loss=1.39382, validation/accuracy=0.687047, validation/bleu=29.6776, validation/loss=1.41016, validation/num_examples=3000 +I0916 14:03:48.214890 139652707104512 logging_writer.py:48] [81500] global_step=81500, grad_norm=0.370178, loss=1.50123 +I0916 14:03:48.218066 139681154073792 submission.py:307] 81500) loss = 1.501, grad_norm = 0.370 +I0916 14:05:06.628833 139652715497216 logging_writer.py:48] [82000] global_step=82000, grad_norm=0.361624, loss=1.45962 +I0916 14:05:06.632148 139681154073792 submission.py:307] 82000) loss = 1.460, grad_norm = 0.362 +I0916 14:06:25.283912 139652707104512 logging_writer.py:48] [82500] global_step=82500, grad_norm=0.372117, loss=1.55606 +I0916 14:06:25.287290 139681154073792 submission.py:307] 82500) loss = 1.556, grad_norm = 0.372 +I0916 14:07:44.054594 139652715497216 logging_writer.py:48] [83000] global_step=83000, grad_norm=0.356875, loss=1.49174 +I0916 14:07:44.057912 139681154073792 submission.py:307] 83000) loss = 1.492, grad_norm = 0.357 +I0916 14:09:02.811851 139652707104512 logging_writer.py:48] [83500] global_step=83500, grad_norm=0.344674, loss=1.55089 +I0916 14:09:02.815608 139681154073792 submission.py:307] 83500) loss = 1.551, grad_norm = 0.345 +I0916 14:10:21.545659 139652715497216 logging_writer.py:48] [84000] global_step=84000, grad_norm=0.334955, loss=1.51322 +I0916 14:10:21.548854 139681154073792 submission.py:307] 84000) loss = 1.513, grad_norm = 0.335 +I0916 14:11:40.334065 139652707104512 logging_writer.py:48] [84500] global_step=84500, grad_norm=0.3433, loss=1.50559 +I0916 14:11:40.337496 139681154073792 submission.py:307] 84500) loss = 1.506, grad_norm = 0.343 +I0916 14:12:59.130356 139652715497216 logging_writer.py:48] [85000] global_step=85000, grad_norm=0.342618, loss=1.43421 +I0916 14:12:59.133623 139681154073792 submission.py:307] 85000) loss = 1.434, grad_norm = 0.343 +I0916 14:14:17.919649 139652707104512 logging_writer.py:48] [85500] global_step=85500, grad_norm=0.372152, loss=1.50584 +I0916 14:14:17.923051 139681154073792 submission.py:307] 85500) loss = 1.506, grad_norm = 0.372 +I0916 14:14:27.787596 139681154073792 spec.py:333] Evaluating on the training split. +I0916 14:14:30.017477 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 14:16:20.729290 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 14:16:22.945739 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 14:18:21.689031 139681154073792 spec.py:363] Evaluating on the test split. +I0916 14:18:23.903843 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 14:20:16.278466 139681154073792 submission_runner.py:516] Time since start: 19365.74s, Step: 85560, {'train/accuracy': 0.6885686927180703, 'train/loss': 1.4019506491952032, 'train/bleu': 34.92511946075621, 'validation/accuracy': 0.6890925097022975, 'validation/loss': 1.4029911671894955, 'validation/bleu': 28.31170686241403, 'validation/num_examples': 3000, 'test/accuracy': 0.7047237232002789, 'test/loss': 1.2981343690081923, 'test/bleu': 30.09348926656585, 'test/num_examples': 3003, 'score': 13535.897929668427, 'total_duration': 19365.742505788803, 'accumulated_submission_time': 13535.897929668427, 'accumulated_eval_time': 5774.228710174561, 'accumulated_logging_time': 0.8166906833648682} +I0916 14:20:16.312556 139652715497216 logging_writer.py:48] [85560] accumulated_eval_time=5774.23, accumulated_logging_time=0.816691, accumulated_submission_time=13535.9, global_step=85560, preemption_count=0, score=13535.9, test/accuracy=0.704724, test/bleu=30.0935, test/loss=1.29813, test/num_examples=3003, total_duration=19365.7, train/accuracy=0.688569, train/bleu=34.9251, train/loss=1.40195, validation/accuracy=0.689093, validation/bleu=28.3117, validation/loss=1.40299, validation/num_examples=3000 +I0916 14:21:26.159177 139652707104512 logging_writer.py:48] [86000] global_step=86000, grad_norm=0.374693, loss=1.5311 +I0916 14:21:26.162403 139681154073792 submission.py:307] 86000) loss = 1.531, grad_norm = 0.375 +I0916 14:22:44.799169 139652715497216 logging_writer.py:48] [86500] global_step=86500, grad_norm=0.395786, loss=1.49463 +I0916 14:22:44.802645 139681154073792 submission.py:307] 86500) loss = 1.495, grad_norm = 0.396 +I0916 14:24:03.524543 139652707104512 logging_writer.py:48] [87000] global_step=87000, grad_norm=0.375521, loss=1.52871 +I0916 14:24:03.527890 139681154073792 submission.py:307] 87000) loss = 1.529, grad_norm = 0.376 +I0916 14:25:22.263180 139652715497216 logging_writer.py:48] [87500] global_step=87500, grad_norm=0.342102, loss=1.36048 +I0916 14:25:22.266478 139681154073792 submission.py:307] 87500) loss = 1.360, grad_norm = 0.342 +I0916 14:26:41.044148 139652707104512 logging_writer.py:48] [88000] global_step=88000, grad_norm=0.372613, loss=1.46203 +I0916 14:26:41.047627 139681154073792 submission.py:307] 88000) loss = 1.462, grad_norm = 0.373 +I0916 14:27:59.770516 139652715497216 logging_writer.py:48] [88500] global_step=88500, grad_norm=0.352, loss=1.38406 +I0916 14:27:59.773866 139681154073792 submission.py:307] 88500) loss = 1.384, grad_norm = 0.352 +I0916 14:29:18.511578 139652707104512 logging_writer.py:48] [89000] global_step=89000, grad_norm=0.362135, loss=1.37459 +I0916 14:29:18.514930 139681154073792 submission.py:307] 89000) loss = 1.375, grad_norm = 0.362 +I0916 14:30:37.289546 139652715497216 logging_writer.py:48] [89500] global_step=89500, grad_norm=0.4065, loss=1.43694 +I0916 14:30:37.292777 139681154073792 submission.py:307] 89500) loss = 1.437, grad_norm = 0.406 +I0916 14:31:01.004155 139681154073792 spec.py:333] Evaluating on the training split. +I0916 14:31:03.235195 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 14:32:41.302954 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 14:32:43.515174 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 14:33:52.824081 139681154073792 spec.py:363] Evaluating on the test split. +I0916 14:33:55.036081 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 14:34:52.700598 139681154073792 submission_runner.py:516] Time since start: 20242.16s, Step: 89648, {'train/accuracy': 0.6970119567453876, 'train/loss': 1.347744750348643, 'train/bleu': 35.79026020455463, 'validation/accuracy': 0.6884353572801329, 'validation/loss': 1.408247805358892, 'validation/bleu': 30.202900919389673, 'validation/num_examples': 3000, 'test/accuracy': 0.7057695659752484, 'test/loss': 1.3033401603625587, 'test/bleu': 30.257084271714383, 'test/num_examples': 3003, 'score': 14178.007321834564, 'total_duration': 20242.164640188217, 'accumulated_submission_time': 14178.007321834564, 'accumulated_eval_time': 6005.925210952759, 'accumulated_logging_time': 0.8602900505065918} +I0916 14:34:52.730101 139652707104512 logging_writer.py:48] [89648] accumulated_eval_time=6005.93, accumulated_logging_time=0.86029, accumulated_submission_time=14178, global_step=89648, preemption_count=0, score=14178, test/accuracy=0.70577, test/bleu=30.2571, test/loss=1.30334, test/num_examples=3003, total_duration=20242.2, train/accuracy=0.697012, train/bleu=35.7903, train/loss=1.34774, validation/accuracy=0.688435, validation/bleu=30.2029, validation/loss=1.40825, validation/num_examples=3000 +I0916 14:35:48.826704 139652715497216 logging_writer.py:48] [90000] global_step=90000, grad_norm=0.336152, loss=1.49889 +I0916 14:35:48.829948 139681154073792 submission.py:307] 90000) loss = 1.499, grad_norm = 0.336 +I0916 14:37:07.452617 139652707104512 logging_writer.py:48] [90500] global_step=90500, grad_norm=0.339219, loss=1.44661 +I0916 14:37:07.456147 139681154073792 submission.py:307] 90500) loss = 1.447, grad_norm = 0.339 +I0916 14:38:26.140205 139652715497216 logging_writer.py:48] [91000] global_step=91000, grad_norm=0.346326, loss=1.48866 +I0916 14:38:26.143538 139681154073792 submission.py:307] 91000) loss = 1.489, grad_norm = 0.346 +I0916 14:39:44.880768 139652707104512 logging_writer.py:48] [91500] global_step=91500, grad_norm=0.372322, loss=1.49845 +I0916 14:39:44.884069 139681154073792 submission.py:307] 91500) loss = 1.498, grad_norm = 0.372 +I0916 14:41:03.622748 139652715497216 logging_writer.py:48] [92000] global_step=92000, grad_norm=0.5, loss=1.48622 +I0916 14:41:03.626111 139681154073792 submission.py:307] 92000) loss = 1.486, grad_norm = 0.500 +I0916 14:42:22.405564 139652707104512 logging_writer.py:48] [92500] global_step=92500, grad_norm=0.344241, loss=1.44309 +I0916 14:42:22.409085 139681154073792 submission.py:307] 92500) loss = 1.443, grad_norm = 0.344 +I0916 14:43:41.160016 139652715497216 logging_writer.py:48] [93000] global_step=93000, grad_norm=0.350094, loss=1.46531 +I0916 14:43:41.163414 139681154073792 submission.py:307] 93000) loss = 1.465, grad_norm = 0.350 +I0916 14:44:59.884663 139652707104512 logging_writer.py:48] [93500] global_step=93500, grad_norm=0.354278, loss=1.4757 +I0916 14:44:59.887883 139681154073792 submission.py:307] 93500) loss = 1.476, grad_norm = 0.354 +I0916 14:45:37.304610 139681154073792 spec.py:333] Evaluating on the training split. +I0916 14:45:39.534498 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 14:47:11.696713 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 14:47:13.907699 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 14:48:15.116042 139681154073792 spec.py:363] Evaluating on the test split. +I0916 14:48:17.330840 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 14:49:18.012537 139681154073792 submission_runner.py:516] Time since start: 21107.48s, Step: 93735, {'train/accuracy': 0.6938509699974864, 'train/loss': 1.3737723290793593, 'train/bleu': 35.663830096058305, 'validation/accuracy': 0.6909895723549615, 'validation/loss': 1.393101139477502, 'validation/bleu': 30.513111798349964, 'validation/num_examples': 3000, 'test/accuracy': 0.7076753239207484, 'test/loss': 1.2846936552204986, 'test/bleu': 30.66273337214052, 'test/num_examples': 3003, 'score': 14819.999969244003, 'total_duration': 21107.47658610344, 'accumulated_submission_time': 14819.999969244003, 'accumulated_eval_time': 6226.633292675018, 'accumulated_logging_time': 0.8993208408355713} +I0916 14:49:18.041407 139652715497216 logging_writer.py:48] [93735] accumulated_eval_time=6226.63, accumulated_logging_time=0.899321, accumulated_submission_time=14820, global_step=93735, preemption_count=0, score=14820, test/accuracy=0.707675, test/bleu=30.6627, test/loss=1.28469, test/num_examples=3003, total_duration=21107.5, train/accuracy=0.693851, train/bleu=35.6638, train/loss=1.37377, validation/accuracy=0.69099, validation/bleu=30.5131, validation/loss=1.3931, validation/num_examples=3000 +I0916 14:50:00.403839 139652707104512 logging_writer.py:48] [94000] global_step=94000, grad_norm=0.347067, loss=1.38791 +I0916 14:50:00.407535 139681154073792 submission.py:307] 94000) loss = 1.388, grad_norm = 0.347 +I0916 14:51:18.950463 139652715497216 logging_writer.py:48] [94500] global_step=94500, grad_norm=0.38658, loss=1.46714 +I0916 14:51:18.953929 139681154073792 submission.py:307] 94500) loss = 1.467, grad_norm = 0.387 +I0916 14:52:37.618825 139652707104512 logging_writer.py:48] [95000] global_step=95000, grad_norm=0.38559, loss=1.49406 +I0916 14:52:37.622311 139681154073792 submission.py:307] 95000) loss = 1.494, grad_norm = 0.386 +I0916 14:53:56.310738 139652715497216 logging_writer.py:48] [95500] global_step=95500, grad_norm=0.366043, loss=1.45 +I0916 14:53:56.314048 139681154073792 submission.py:307] 95500) loss = 1.450, grad_norm = 0.366 +I0916 14:55:15.014302 139652707104512 logging_writer.py:48] [96000] global_step=96000, grad_norm=0.396879, loss=1.44603 +I0916 14:55:15.017521 139681154073792 submission.py:307] 96000) loss = 1.446, grad_norm = 0.397 +I0916 14:56:33.717087 139652715497216 logging_writer.py:48] [96500] global_step=96500, grad_norm=0.339498, loss=1.42564 +I0916 14:56:33.720601 139681154073792 submission.py:307] 96500) loss = 1.426, grad_norm = 0.339 +I0916 14:57:52.455774 139652707104512 logging_writer.py:48] [97000] global_step=97000, grad_norm=0.353777, loss=1.46426 +I0916 14:57:52.459244 139681154073792 submission.py:307] 97000) loss = 1.464, grad_norm = 0.354 +I0916 14:59:11.182440 139652715497216 logging_writer.py:48] [97500] global_step=97500, grad_norm=0.422317, loss=1.47947 +I0916 14:59:11.185667 139681154073792 submission.py:307] 97500) loss = 1.479, grad_norm = 0.422 +I0916 15:00:02.653424 139681154073792 spec.py:333] Evaluating on the training split. +I0916 15:00:04.883846 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 15:02:08.132742 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 15:02:10.340104 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 15:04:00.117610 139681154073792 spec.py:363] Evaluating on the test split. +I0916 15:04:02.333162 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 15:05:59.492019 139681154073792 submission_runner.py:516] Time since start: 22108.96s, Step: 97824, {'train/accuracy': 0.6972819389292072, 'train/loss': 1.3497797352115568, 'train/bleu': 35.71535766677376, 'validation/accuracy': 0.6925146619384757, 'validation/loss': 1.390379052181622, 'validation/bleu': 29.095635009310403, 'validation/num_examples': 3000, 'test/accuracy': 0.7093835337865319, 'test/loss': 1.2811286205043286, 'test/bleu': 30.381134391662084, 'test/num_examples': 3003, 'score': 15462.022831916809, 'total_duration': 22108.956053495407, 'accumulated_submission_time': 15462.022831916809, 'accumulated_eval_time': 6583.471929550171, 'accumulated_logging_time': 0.9379317760467529} +I0916 15:05:59.522888 139652707104512 logging_writer.py:48] [97824] accumulated_eval_time=6583.47, accumulated_logging_time=0.937932, accumulated_submission_time=15462, global_step=97824, preemption_count=0, score=15462, test/accuracy=0.709384, test/bleu=30.3811, test/loss=1.28113, test/num_examples=3003, total_duration=22109, train/accuracy=0.697282, train/bleu=35.7154, train/loss=1.34978, validation/accuracy=0.692515, validation/bleu=29.0956, validation/loss=1.39038, validation/num_examples=3000 +I0916 15:06:27.857698 139652715497216 logging_writer.py:48] [98000] global_step=98000, grad_norm=0.352456, loss=1.47233 +I0916 15:06:27.860813 139681154073792 submission.py:307] 98000) loss = 1.472, grad_norm = 0.352 +I0916 15:07:46.380179 139652707104512 logging_writer.py:48] [98500] global_step=98500, grad_norm=0.349116, loss=1.36102 +I0916 15:07:46.383744 139681154073792 submission.py:307] 98500) loss = 1.361, grad_norm = 0.349 +I0916 15:09:05.082286 139652715497216 logging_writer.py:48] [99000] global_step=99000, grad_norm=0.353305, loss=1.49643 +I0916 15:09:05.085581 139681154073792 submission.py:307] 99000) loss = 1.496, grad_norm = 0.353 +I0916 15:10:23.846457 139652707104512 logging_writer.py:48] [99500] global_step=99500, grad_norm=0.34534, loss=1.37736 +I0916 15:10:23.849910 139681154073792 submission.py:307] 99500) loss = 1.377, grad_norm = 0.345 +I0916 15:11:42.634169 139652715497216 logging_writer.py:48] [100000] global_step=100000, grad_norm=0.352244, loss=1.41151 +I0916 15:11:42.637926 139681154073792 submission.py:307] 100000) loss = 1.412, grad_norm = 0.352 +I0916 15:13:01.382805 139652707104512 logging_writer.py:48] [100500] global_step=100500, grad_norm=0.351119, loss=1.43645 +I0916 15:13:01.386204 139681154073792 submission.py:307] 100500) loss = 1.436, grad_norm = 0.351 +I0916 15:14:20.154297 139652715497216 logging_writer.py:48] [101000] global_step=101000, grad_norm=0.352115, loss=1.41935 +I0916 15:14:20.157499 139681154073792 submission.py:307] 101000) loss = 1.419, grad_norm = 0.352 +I0916 15:15:38.914130 139652707104512 logging_writer.py:48] [101500] global_step=101500, grad_norm=0.331886, loss=1.39526 +I0916 15:15:38.917493 139681154073792 submission.py:307] 101500) loss = 1.395, grad_norm = 0.332 +I0916 15:16:44.085682 139681154073792 spec.py:333] Evaluating on the training split. +I0916 15:16:46.313388 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 15:18:34.883403 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 15:18:37.099495 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 15:20:02.796049 139681154073792 spec.py:363] Evaluating on the test split. +I0916 15:20:05.013073 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 15:21:27.749743 139681154073792 submission_runner.py:516] Time since start: 23037.21s, Step: 101911, {'train/accuracy': 0.6997586999531124, 'train/loss': 1.336101624772709, 'train/bleu': 36.22656518179386, 'validation/accuracy': 0.6933578008952151, 'validation/loss': 1.3865799866089694, 'validation/bleu': 30.279620503488072, 'validation/num_examples': 3000, 'test/accuracy': 0.7086514438440532, 'test/loss': 1.2787330776828771, 'test/bleu': 30.30002914573537, 'test/num_examples': 3003, 'score': 16103.99957895279, 'total_duration': 23037.213791370392, 'accumulated_submission_time': 16103.99957895279, 'accumulated_eval_time': 6867.136032581329, 'accumulated_logging_time': 0.9784302711486816} +I0916 15:21:27.781109 139652715497216 logging_writer.py:48] [101911] accumulated_eval_time=6867.14, accumulated_logging_time=0.97843, accumulated_submission_time=16104, global_step=101911, preemption_count=0, score=16104, test/accuracy=0.708651, test/bleu=30.3, test/loss=1.27873, test/num_examples=3003, total_duration=23037.2, train/accuracy=0.699759, train/bleu=36.2266, train/loss=1.3361, validation/accuracy=0.693358, validation/bleu=30.2796, validation/loss=1.38658, validation/num_examples=3000 +I0916 15:21:42.527179 139652707104512 logging_writer.py:48] [102000] global_step=102000, grad_norm=0.337336, loss=1.41169 +I0916 15:21:42.530337 139681154073792 submission.py:307] 102000) loss = 1.412, grad_norm = 0.337 +I0916 15:23:01.000348 139652715497216 logging_writer.py:48] [102500] global_step=102500, grad_norm=0.330501, loss=1.41832 +I0916 15:23:01.003766 139681154073792 submission.py:307] 102500) loss = 1.418, grad_norm = 0.331 +I0916 15:24:19.676898 139652707104512 logging_writer.py:48] [103000] global_step=103000, grad_norm=0.347031, loss=1.47237 +I0916 15:24:19.680413 139681154073792 submission.py:307] 103000) loss = 1.472, grad_norm = 0.347 +I0916 15:25:38.408955 139652715497216 logging_writer.py:48] [103500] global_step=103500, grad_norm=0.366277, loss=1.44768 +I0916 15:25:38.412399 139681154073792 submission.py:307] 103500) loss = 1.448, grad_norm = 0.366 +I0916 15:26:57.143166 139652707104512 logging_writer.py:48] [104000] global_step=104000, grad_norm=0.327805, loss=1.33232 +I0916 15:26:57.146375 139681154073792 submission.py:307] 104000) loss = 1.332, grad_norm = 0.328 +I0916 15:28:15.929154 139652715497216 logging_writer.py:48] [104500] global_step=104500, grad_norm=0.364682, loss=1.49296 +I0916 15:28:15.932654 139681154073792 submission.py:307] 104500) loss = 1.493, grad_norm = 0.365 +I0916 15:29:34.669427 139652707104512 logging_writer.py:48] [105000] global_step=105000, grad_norm=0.342473, loss=1.41608 +I0916 15:29:34.672673 139681154073792 submission.py:307] 105000) loss = 1.416, grad_norm = 0.342 +I0916 15:30:53.413464 139652715497216 logging_writer.py:48] [105500] global_step=105500, grad_norm=0.337907, loss=1.41644 +I0916 15:30:53.416808 139681154073792 submission.py:307] 105500) loss = 1.416, grad_norm = 0.338 +I0916 15:32:12.420685 139681154073792 spec.py:333] Evaluating on the training split. +I0916 15:32:14.649881 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 15:33:45.231165 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 15:33:47.444952 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 15:35:16.026102 139681154073792 spec.py:363] Evaluating on the test split. +I0916 15:35:18.239791 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 15:36:37.261880 139681154073792 submission_runner.py:516] Time since start: 23946.73s, Step: 105999, {'train/accuracy': 0.7005885488132634, 'train/loss': 1.3351153460701042, 'train/bleu': 36.09823027073485, 'validation/accuracy': 0.6935685856344, 'validation/loss': 1.3848594174281783, 'validation/bleu': 30.316039877116662, 'validation/num_examples': 3000, 'test/accuracy': 0.7098018708965197, 'test/loss': 1.27362170269014, 'test/bleu': 30.551579303481194, 'test/num_examples': 3003, 'score': 16746.064346313477, 'total_duration': 23946.72591495514, 'accumulated_submission_time': 16746.064346313477, 'accumulated_eval_time': 7131.977272987366, 'accumulated_logging_time': 1.0208284854888916} +I0916 15:36:37.292508 139652707104512 logging_writer.py:48] [105999] accumulated_eval_time=7131.98, accumulated_logging_time=1.02083, accumulated_submission_time=16746.1, global_step=105999, preemption_count=0, score=16746.1, test/accuracy=0.709802, test/bleu=30.5516, test/loss=1.27362, test/num_examples=3003, total_duration=23946.7, train/accuracy=0.700589, train/bleu=36.0982, train/loss=1.33512, validation/accuracy=0.693569, validation/bleu=30.316, validation/loss=1.38486, validation/num_examples=3000 +I0916 15:36:38.251370 139652715497216 logging_writer.py:48] [106000] global_step=106000, grad_norm=0.342752, loss=1.43177 +I0916 15:36:38.254402 139681154073792 submission.py:307] 106000) loss = 1.432, grad_norm = 0.343 +I0916 15:37:56.637851 139652707104512 logging_writer.py:48] [106500] global_step=106500, grad_norm=0.339243, loss=1.37924 +I0916 15:37:56.641302 139681154073792 submission.py:307] 106500) loss = 1.379, grad_norm = 0.339 +I0916 15:39:15.277057 139652715497216 logging_writer.py:48] [107000] global_step=107000, grad_norm=0.343458, loss=1.42066 +I0916 15:39:15.280236 139681154073792 submission.py:307] 107000) loss = 1.421, grad_norm = 0.343 +I0916 15:40:34.025975 139652707104512 logging_writer.py:48] [107500] global_step=107500, grad_norm=0.33511, loss=1.41638 +I0916 15:40:34.029586 139681154073792 submission.py:307] 107500) loss = 1.416, grad_norm = 0.335 +I0916 15:41:52.753737 139652715497216 logging_writer.py:48] [108000] global_step=108000, grad_norm=0.36836, loss=1.45404 +I0916 15:41:52.757202 139681154073792 submission.py:307] 108000) loss = 1.454, grad_norm = 0.368 +I0916 15:43:11.486603 139652707104512 logging_writer.py:48] [108500] global_step=108500, grad_norm=0.347886, loss=1.41975 +I0916 15:43:11.490372 139681154073792 submission.py:307] 108500) loss = 1.420, grad_norm = 0.348 +I0916 15:44:30.236647 139652715497216 logging_writer.py:48] [109000] global_step=109000, grad_norm=0.326006, loss=1.46508 +I0916 15:44:30.239784 139681154073792 submission.py:307] 109000) loss = 1.465, grad_norm = 0.326 +I0916 15:45:48.970617 139652707104512 logging_writer.py:48] [109500] global_step=109500, grad_norm=0.341091, loss=1.46629 +I0916 15:45:48.974138 139681154073792 submission.py:307] 109500) loss = 1.466, grad_norm = 0.341 +I0916 15:47:07.709902 139652715497216 logging_writer.py:48] [110000] global_step=110000, grad_norm=0.337047, loss=1.3672 +I0916 15:47:07.713269 139681154073792 submission.py:307] 110000) loss = 1.367, grad_norm = 0.337 +I0916 15:47:21.845198 139681154073792 spec.py:333] Evaluating on the training split. +I0916 15:47:24.075757 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 15:49:04.570533 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 15:49:06.779708 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 15:50:24.824424 139681154073792 spec.py:363] Evaluating on the test split. +I0916 15:50:27.038534 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 15:51:35.671597 139681154073792 submission_runner.py:516] Time since start: 24845.14s, Step: 110087, {'train/accuracy': 0.7035749922581978, 'train/loss': 1.309232314282765, 'train/bleu': 36.38694155896813, 'validation/accuracy': 0.6942009398519547, 'validation/loss': 1.3826231230858885, 'validation/bleu': 30.652193336253045, 'validation/num_examples': 3000, 'test/accuracy': 0.7104293765615014, 'test/loss': 1.271752258729882, 'test/bleu': 30.433257472134322, 'test/num_examples': 3003, 'score': 17388.03769659996, 'total_duration': 24845.135648965836, 'accumulated_submission_time': 17388.03769659996, 'accumulated_eval_time': 7385.803707122803, 'accumulated_logging_time': 1.0621931552886963} +I0916 15:51:35.704199 139652707104512 logging_writer.py:48] [110087] accumulated_eval_time=7385.8, accumulated_logging_time=1.06219, accumulated_submission_time=17388, global_step=110087, preemption_count=0, score=17388, test/accuracy=0.710429, test/bleu=30.4333, test/loss=1.27175, test/num_examples=3003, total_duration=24845.1, train/accuracy=0.703575, train/bleu=36.3869, train/loss=1.30923, validation/accuracy=0.694201, validation/bleu=30.6522, validation/loss=1.38262, validation/num_examples=3000 +I0916 15:52:41.272737 139652715497216 logging_writer.py:48] [110500] global_step=110500, grad_norm=0.35106, loss=1.39583 +I0916 15:52:41.276319 139681154073792 submission.py:307] 110500) loss = 1.396, grad_norm = 0.351 +I0916 15:53:59.904882 139652707104512 logging_writer.py:48] [111000] global_step=111000, grad_norm=0.324638, loss=1.31988 +I0916 15:53:59.908385 139681154073792 submission.py:307] 111000) loss = 1.320, grad_norm = 0.325 +I0916 15:55:18.646220 139652715497216 logging_writer.py:48] [111500] global_step=111500, grad_norm=0.335104, loss=1.39458 +I0916 15:55:18.649481 139681154073792 submission.py:307] 111500) loss = 1.395, grad_norm = 0.335 +I0916 15:56:37.427482 139652707104512 logging_writer.py:48] [112000] global_step=112000, grad_norm=0.347456, loss=1.42725 +I0916 15:56:37.430855 139681154073792 submission.py:307] 112000) loss = 1.427, grad_norm = 0.347 +I0916 15:57:56.178990 139652715497216 logging_writer.py:48] [112500] global_step=112500, grad_norm=0.329691, loss=1.38878 +I0916 15:57:56.182492 139681154073792 submission.py:307] 112500) loss = 1.389, grad_norm = 0.330 +I0916 15:59:14.998687 139652707104512 logging_writer.py:48] [113000] global_step=113000, grad_norm=0.338291, loss=1.38996 +I0916 15:59:15.002180 139681154073792 submission.py:307] 113000) loss = 1.390, grad_norm = 0.338 +I0916 16:00:33.835887 139652715497216 logging_writer.py:48] [113500] global_step=113500, grad_norm=0.339911, loss=1.40086 +I0916 16:00:33.839107 139681154073792 submission.py:307] 113500) loss = 1.401, grad_norm = 0.340 +I0916 16:01:52.673093 139652707104512 logging_writer.py:48] [114000] global_step=114000, grad_norm=0.336272, loss=1.36955 +I0916 16:01:52.676404 139681154073792 submission.py:307] 114000) loss = 1.370, grad_norm = 0.336 +I0916 16:02:20.383245 139681154073792 spec.py:333] Evaluating on the training split. +I0916 16:02:22.614312 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 16:03:56.239109 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 16:03:58.452909 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 16:05:19.493283 139681154073792 spec.py:363] Evaluating on the test split. +I0916 16:05:21.711867 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 16:06:41.635123 139681154073792 submission_runner.py:516] Time since start: 25751.10s, Step: 114173, {'train/accuracy': 0.7029496215087444, 'train/loss': 1.3174546640393585, 'train/bleu': 36.057146377560564, 'validation/accuracy': 0.693680177555145, 'validation/loss': 1.3834639372109458, 'validation/bleu': 30.78495669829378, 'validation/num_examples': 3000, 'test/accuracy': 0.710499099413166, 'test/loss': 1.2717316505432572, 'test/bleu': 30.540808370113545, 'test/num_examples': 3003, 'score': 18030.135451316833, 'total_duration': 25751.099177598953, 'accumulated_submission_time': 18030.135451316833, 'accumulated_eval_time': 7647.055639743805, 'accumulated_logging_time': 1.1045951843261719} +I0916 16:06:41.666157 139652715497216 logging_writer.py:48] [114173] accumulated_eval_time=7647.06, accumulated_logging_time=1.1046, accumulated_submission_time=18030.1, global_step=114173, preemption_count=0, score=18030.1, test/accuracy=0.710499, test/bleu=30.5408, test/loss=1.27173, test/num_examples=3003, total_duration=25751.1, train/accuracy=0.70295, train/bleu=36.0571, train/loss=1.31745, validation/accuracy=0.69368, validation/bleu=30.785, validation/loss=1.38346, validation/num_examples=3000 +I0916 16:07:33.745551 139652707104512 logging_writer.py:48] [114500] global_step=114500, grad_norm=0.362425, loss=1.46924 +I0916 16:07:33.748758 139681154073792 submission.py:307] 114500) loss = 1.469, grad_norm = 0.362 +I0916 16:08:52.344914 139652715497216 logging_writer.py:48] [115000] global_step=115000, grad_norm=0.390637, loss=1.44547 +I0916 16:08:52.348339 139681154073792 submission.py:307] 115000) loss = 1.445, grad_norm = 0.391 +I0916 16:10:11.023037 139652707104512 logging_writer.py:48] [115500] global_step=115500, grad_norm=0.339732, loss=1.46078 +I0916 16:10:11.026407 139681154073792 submission.py:307] 115500) loss = 1.461, grad_norm = 0.340 +I0916 16:11:29.765794 139652715497216 logging_writer.py:48] [116000] global_step=116000, grad_norm=0.360638, loss=1.43251 +I0916 16:11:29.769320 139681154073792 submission.py:307] 116000) loss = 1.433, grad_norm = 0.361 +I0916 16:12:48.544332 139652707104512 logging_writer.py:48] [116500] global_step=116500, grad_norm=0.328048, loss=1.38748 +I0916 16:12:48.547610 139681154073792 submission.py:307] 116500) loss = 1.387, grad_norm = 0.328 +I0916 16:14:07.309709 139652715497216 logging_writer.py:48] [117000] global_step=117000, grad_norm=0.342283, loss=1.44554 +I0916 16:14:07.313150 139681154073792 submission.py:307] 117000) loss = 1.446, grad_norm = 0.342 +I0916 16:15:26.161230 139652707104512 logging_writer.py:48] [117500] global_step=117500, grad_norm=0.339542, loss=1.36011 +I0916 16:15:26.164746 139681154073792 submission.py:307] 117500) loss = 1.360, grad_norm = 0.340 +I0916 16:16:44.964291 139652715497216 logging_writer.py:48] [118000] global_step=118000, grad_norm=0.327527, loss=1.37907 +I0916 16:16:44.967782 139681154073792 submission.py:307] 118000) loss = 1.379, grad_norm = 0.328 +I0916 16:17:26.363331 139681154073792 spec.py:333] Evaluating on the training split. +I0916 16:17:28.595662 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 16:19:16.852372 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 16:19:19.068987 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 16:20:48.090686 139681154073792 spec.py:363] Evaluating on the test split. +I0916 16:20:50.301279 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 16:22:14.772174 139681154073792 submission_runner.py:516] Time since start: 26684.24s, Step: 118260, {'train/accuracy': 0.7059680927383081, 'train/loss': 1.3004796195497454, 'train/bleu': 36.6821103814912, 'validation/accuracy': 0.6938413658851099, 'validation/loss': 1.381956768204982, 'validation/bleu': 30.666652184073417, 'validation/num_examples': 3000, 'test/accuracy': 0.7104874789378886, 'test/loss': 1.270655667160537, 'test/bleu': 30.5197138290683, 'test/num_examples': 3003, 'score': 18672.254566431046, 'total_duration': 26684.236224412918, 'accumulated_submission_time': 18672.254566431046, 'accumulated_eval_time': 7935.464534521103, 'accumulated_logging_time': 1.1450221538543701} +I0916 16:22:14.807401 139652707104512 logging_writer.py:48] [118260] accumulated_eval_time=7935.46, accumulated_logging_time=1.14502, accumulated_submission_time=18672.3, global_step=118260, preemption_count=0, score=18672.3, test/accuracy=0.710487, test/bleu=30.5197, test/loss=1.27066, test/num_examples=3003, total_duration=26684.2, train/accuracy=0.705968, train/bleu=36.6821, train/loss=1.30048, validation/accuracy=0.693841, validation/bleu=30.6667, validation/loss=1.38196, validation/num_examples=3000 +I0916 16:22:53.297399 139652715497216 logging_writer.py:48] [118500] global_step=118500, grad_norm=0.372885, loss=1.3963 +I0916 16:22:53.300646 139681154073792 submission.py:307] 118500) loss = 1.396, grad_norm = 0.373 +I0916 16:24:11.933828 139652707104512 logging_writer.py:48] [119000] global_step=119000, grad_norm=0.342928, loss=1.41449 +I0916 16:24:11.937184 139681154073792 submission.py:307] 119000) loss = 1.414, grad_norm = 0.343 +I0916 16:25:30.670437 139652715497216 logging_writer.py:48] [119500] global_step=119500, grad_norm=0.347265, loss=1.35486 +I0916 16:25:30.673759 139681154073792 submission.py:307] 119500) loss = 1.355, grad_norm = 0.347 +I0916 16:26:49.430137 139652707104512 logging_writer.py:48] [120000] global_step=120000, grad_norm=0.331056, loss=1.45731 +I0916 16:26:49.433609 139681154073792 submission.py:307] 120000) loss = 1.457, grad_norm = 0.331 +I0916 16:28:08.179453 139652715497216 logging_writer.py:48] [120500] global_step=120500, grad_norm=0.333503, loss=1.40414 +I0916 16:28:08.182788 139681154073792 submission.py:307] 120500) loss = 1.404, grad_norm = 0.334 +I0916 16:29:26.903311 139652707104512 logging_writer.py:48] [121000] global_step=121000, grad_norm=0.352082, loss=1.41599 +I0916 16:29:26.906705 139681154073792 submission.py:307] 121000) loss = 1.416, grad_norm = 0.352 +I0916 16:30:45.659269 139652715497216 logging_writer.py:48] [121500] global_step=121500, grad_norm=0.345132, loss=1.38272 +I0916 16:30:45.662577 139681154073792 submission.py:307] 121500) loss = 1.383, grad_norm = 0.345 +I0916 16:32:04.523471 139652707104512 logging_writer.py:48] [122000] global_step=122000, grad_norm=0.329676, loss=1.44874 +I0916 16:32:04.526823 139681154073792 submission.py:307] 122000) loss = 1.449, grad_norm = 0.330 +I0916 16:32:59.440095 139681154073792 spec.py:333] Evaluating on the training split. +I0916 16:33:01.674234 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 16:34:39.309676 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 16:34:41.521103 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 16:36:05.207390 139681154073792 spec.py:363] Evaluating on the test split. +I0916 16:36:07.422432 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 16:37:32.414929 139681154073792 submission_runner.py:516] Time since start: 27601.88s, Step: 122346, {'train/accuracy': 0.6997066171583122, 'train/loss': 1.3324967338238327, 'train/bleu': 36.64013427257552, 'validation/accuracy': 0.6938537649874149, 'validation/loss': 1.382162225204895, 'validation/bleu': 30.657045900752422, 'validation/num_examples': 3000, 'test/accuracy': 0.7104061356109465, 'test/loss': 1.2707375551972575, 'test/bleu': 30.456589562506334, 'test/num_examples': 3003, 'score': 19314.32937502861, 'total_duration': 27601.878966331482, 'accumulated_submission_time': 19314.32937502861, 'accumulated_eval_time': 8208.439351797104, 'accumulated_logging_time': 1.1900050640106201} +I0916 16:37:32.447208 139652715497216 logging_writer.py:48] [122346] accumulated_eval_time=8208.44, accumulated_logging_time=1.19001, accumulated_submission_time=19314.3, global_step=122346, preemption_count=0, score=19314.3, test/accuracy=0.710406, test/bleu=30.4566, test/loss=1.27074, test/num_examples=3003, total_duration=27601.9, train/accuracy=0.699707, train/bleu=36.6401, train/loss=1.3325, validation/accuracy=0.693854, validation/bleu=30.657, validation/loss=1.38216, validation/num_examples=3000 +I0916 16:37:57.390114 139652707104512 logging_writer.py:48] [122500] global_step=122500, grad_norm=0.324482, loss=1.36099 +I0916 16:37:57.393284 139681154073792 submission.py:307] 122500) loss = 1.361, grad_norm = 0.324 +I0916 16:39:15.962962 139652715497216 logging_writer.py:48] [123000] global_step=123000, grad_norm=0.325718, loss=1.43563 +I0916 16:39:15.966728 139681154073792 submission.py:307] 123000) loss = 1.436, grad_norm = 0.326 +I0916 16:40:34.666803 139652707104512 logging_writer.py:48] [123500] global_step=123500, grad_norm=0.373255, loss=1.38954 +I0916 16:40:34.670162 139681154073792 submission.py:307] 123500) loss = 1.390, grad_norm = 0.373 +I0916 16:41:53.462689 139652715497216 logging_writer.py:48] [124000] global_step=124000, grad_norm=0.360969, loss=1.39975 +I0916 16:41:53.465944 139681154073792 submission.py:307] 124000) loss = 1.400, grad_norm = 0.361 +I0916 16:43:12.269054 139652707104512 logging_writer.py:48] [124500] global_step=124500, grad_norm=0.336276, loss=1.4129 +I0916 16:43:12.272485 139681154073792 submission.py:307] 124500) loss = 1.413, grad_norm = 0.336 +I0916 16:44:31.019236 139652715497216 logging_writer.py:48] [125000] global_step=125000, grad_norm=0.353073, loss=1.36884 +I0916 16:44:31.022525 139681154073792 submission.py:307] 125000) loss = 1.369, grad_norm = 0.353 +I0916 16:45:49.794778 139652707104512 logging_writer.py:48] [125500] global_step=125500, grad_norm=0.332438, loss=1.38063 +I0916 16:45:49.798020 139681154073792 submission.py:307] 125500) loss = 1.381, grad_norm = 0.332 +I0916 16:47:08.563057 139652715497216 logging_writer.py:48] [126000] global_step=126000, grad_norm=0.337209, loss=1.35673 +I0916 16:47:08.566511 139681154073792 submission.py:307] 126000) loss = 1.357, grad_norm = 0.337 +I0916 16:48:17.029219 139681154073792 spec.py:333] Evaluating on the training split. +I0916 16:48:19.258173 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 16:50:01.839676 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 16:50:04.058343 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 16:51:22.180994 139681154073792 spec.py:363] Evaluating on the test split. +I0916 16:51:24.399098 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 16:52:43.073993 139681154073792 submission_runner.py:516] Time since start: 28512.54s, Step: 126432, {'train/accuracy': 0.7062779241608431, 'train/loss': 1.2974285606885096, 'train/bleu': 36.67276794079318, 'validation/accuracy': 0.6939033613966349, 'validation/loss': 1.3826583830330683, 'validation/bleu': 30.73521655991802, 'validation/num_examples': 3000, 'test/accuracy': 0.7104874789378886, 'test/loss': 1.270744908779269, 'test/bleu': 30.44486588346862, 'test/num_examples': 3003, 'score': 19956.34680557251, 'total_duration': 28512.538029670715, 'accumulated_submission_time': 19956.34680557251, 'accumulated_eval_time': 8474.484276294708, 'accumulated_logging_time': 1.2323086261749268} +I0916 16:52:43.106676 139652707104512 logging_writer.py:48] [126432] accumulated_eval_time=8474.48, accumulated_logging_time=1.23231, accumulated_submission_time=19956.3, global_step=126432, preemption_count=0, score=19956.3, test/accuracy=0.710487, test/bleu=30.4449, test/loss=1.27074, test/num_examples=3003, total_duration=28512.5, train/accuracy=0.706278, train/bleu=36.6728, train/loss=1.29743, validation/accuracy=0.693903, validation/bleu=30.7352, validation/loss=1.38266, validation/num_examples=3000 +I0916 16:52:54.548852 139652715497216 logging_writer.py:48] [126500] global_step=126500, grad_norm=0.319102, loss=1.40902 +I0916 16:52:54.551862 139681154073792 submission.py:307] 126500) loss = 1.409, grad_norm = 0.319 +I0916 16:54:12.967350 139652707104512 logging_writer.py:48] [127000] global_step=127000, grad_norm=0.320461, loss=1.37844 +I0916 16:54:12.970701 139681154073792 submission.py:307] 127000) loss = 1.378, grad_norm = 0.320 +I0916 16:55:31.726310 139652715497216 logging_writer.py:48] [127500] global_step=127500, grad_norm=0.341896, loss=1.4114 +I0916 16:55:31.729753 139681154073792 submission.py:307] 127500) loss = 1.411, grad_norm = 0.342 +I0916 16:56:50.527116 139652707104512 logging_writer.py:48] [128000] global_step=128000, grad_norm=0.350053, loss=1.42834 +I0916 16:56:50.530390 139681154073792 submission.py:307] 128000) loss = 1.428, grad_norm = 0.350 +I0916 16:58:09.291335 139652715497216 logging_writer.py:48] [128500] global_step=128500, grad_norm=0.346905, loss=1.39558 +I0916 16:58:09.294513 139681154073792 submission.py:307] 128500) loss = 1.396, grad_norm = 0.347 +I0916 16:59:28.109385 139652707104512 logging_writer.py:48] [129000] global_step=129000, grad_norm=0.334782, loss=1.4148 +I0916 16:59:28.112973 139681154073792 submission.py:307] 129000) loss = 1.415, grad_norm = 0.335 +I0916 17:00:46.931409 139652715497216 logging_writer.py:48] [129500] global_step=129500, grad_norm=0.366185, loss=1.41545 +I0916 17:00:46.934844 139681154073792 submission.py:307] 129500) loss = 1.415, grad_norm = 0.366 +I0916 17:02:05.763740 139652707104512 logging_writer.py:48] [130000] global_step=130000, grad_norm=0.345821, loss=1.41759 +I0916 17:02:05.766945 139681154073792 submission.py:307] 130000) loss = 1.418, grad_norm = 0.346 +I0916 17:03:24.588861 139652715497216 logging_writer.py:48] [130500] global_step=130500, grad_norm=0.350834, loss=1.42643 +I0916 17:03:24.592336 139681154073792 submission.py:307] 130500) loss = 1.426, grad_norm = 0.351 +I0916 17:03:27.688878 139681154073792 spec.py:333] Evaluating on the training split. +I0916 17:03:29.924123 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 17:05:05.376290 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 17:05:07.587846 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 17:06:31.189490 139681154073792 spec.py:363] Evaluating on the test split. +I0916 17:06:33.401492 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 17:07:52.696457 139681154073792 submission_runner.py:516] Time since start: 29422.16s, Step: 130517, {'train/accuracy': 0.7066352230057698, 'train/loss': 1.2944492899418445, 'train/bleu': 36.1449996008607, 'validation/accuracy': 0.6938537649874149, 'validation/loss': 1.3822447567296128, 'validation/bleu': 30.70360554311707, 'validation/num_examples': 3000, 'test/accuracy': 0.7105572017895532, 'test/loss': 1.271061203590727, 'test/bleu': 30.341595634642246, 'test/num_examples': 3003, 'score': 20598.373563289642, 'total_duration': 29422.16049194336, 'accumulated_submission_time': 20598.373563289642, 'accumulated_eval_time': 8739.491944551468, 'accumulated_logging_time': 1.2747008800506592} +I0916 17:07:52.728968 139652707104512 logging_writer.py:48] [130517] accumulated_eval_time=8739.49, accumulated_logging_time=1.2747, accumulated_submission_time=20598.4, global_step=130517, preemption_count=0, score=20598.4, test/accuracy=0.710557, test/bleu=30.3416, test/loss=1.27106, test/num_examples=3003, total_duration=29422.2, train/accuracy=0.706635, train/bleu=36.145, train/loss=1.29445, validation/accuracy=0.693854, validation/bleu=30.7036, validation/loss=1.38224, validation/num_examples=3000 +I0916 17:09:09.328234 139652715497216 logging_writer.py:48] [131000] global_step=131000, grad_norm=0.350923, loss=1.40041 +I0916 17:09:09.331575 139681154073792 submission.py:307] 131000) loss = 1.400, grad_norm = 0.351 +I0916 17:10:28.036837 139652707104512 logging_writer.py:48] [131500] global_step=131500, grad_norm=0.367686, loss=1.37493 +I0916 17:10:28.040026 139681154073792 submission.py:307] 131500) loss = 1.375, grad_norm = 0.368 +I0916 17:11:46.780614 139652715497216 logging_writer.py:48] [132000] global_step=132000, grad_norm=0.334628, loss=1.36576 +I0916 17:11:46.784290 139681154073792 submission.py:307] 132000) loss = 1.366, grad_norm = 0.335 +I0916 17:13:05.594178 139652707104512 logging_writer.py:48] [132500] global_step=132500, grad_norm=0.355306, loss=1.30801 +I0916 17:13:05.597597 139681154073792 submission.py:307] 132500) loss = 1.308, grad_norm = 0.355 +I0916 17:14:24.396784 139652715497216 logging_writer.py:48] [133000] global_step=133000, grad_norm=0.380181, loss=1.49909 +I0916 17:14:24.400091 139681154073792 submission.py:307] 133000) loss = 1.499, grad_norm = 0.380 +I0916 17:15:43.252269 139652707104512 logging_writer.py:48] [133500] global_step=133500, grad_norm=0.360857, loss=1.49967 +I0916 17:15:43.255659 139681154073792 submission.py:307] 133500) loss = 1.500, grad_norm = 0.361 +I0916 17:17:02.129863 139652715497216 logging_writer.py:48] [134000] global_step=134000, grad_norm=0.332743, loss=1.42026 +I0916 17:17:02.133275 139681154073792 submission.py:307] 134000) loss = 1.420, grad_norm = 0.333 +I0916 17:18:21.008573 139652707104512 logging_writer.py:48] [134500] global_step=134500, grad_norm=0.35281, loss=1.36861 +I0916 17:18:21.011822 139681154073792 submission.py:307] 134500) loss = 1.369, grad_norm = 0.353 +I0916 17:18:37.364730 139681154073792 spec.py:333] Evaluating on the training split. +I0916 17:18:39.595981 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 17:20:24.655812 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 17:20:26.872438 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 17:21:42.709248 139681154073792 spec.py:363] Evaluating on the test split. +I0916 17:21:44.921715 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 17:22:55.505477 139681154073792 submission_runner.py:516] Time since start: 30324.97s, Step: 134601, {'train/accuracy': 0.7052066596013812, 'train/loss': 1.303047331715155, 'train/bleu': 36.55484407583282, 'validation/accuracy': 0.6939033613966349, 'validation/loss': 1.382166681132286, 'validation/bleu': 30.8441882006884, 'validation/num_examples': 3000, 'test/accuracy': 0.7101504851548428, 'test/loss': 1.2720226163500088, 'test/bleu': 30.57973386492272, 'test/num_examples': 3003, 'score': 21240.45601129532, 'total_duration': 30324.969521284103, 'accumulated_submission_time': 21240.45601129532, 'accumulated_eval_time': 8997.632767438889, 'accumulated_logging_time': 1.3170082569122314} +I0916 17:22:55.539471 139652715497216 logging_writer.py:48] [134601] accumulated_eval_time=8997.63, accumulated_logging_time=1.31701, accumulated_submission_time=21240.5, global_step=134601, preemption_count=0, score=21240.5, test/accuracy=0.71015, test/bleu=30.5797, test/loss=1.27202, test/num_examples=3003, total_duration=30325, train/accuracy=0.705207, train/bleu=36.5548, train/loss=1.30305, validation/accuracy=0.693903, validation/bleu=30.8442, validation/loss=1.38217, validation/num_examples=3000 +I0916 17:23:58.848142 139652707104512 logging_writer.py:48] [135000] global_step=135000, grad_norm=0.352753, loss=1.34008 +I0916 17:23:58.851446 139681154073792 submission.py:307] 135000) loss = 1.340, grad_norm = 0.353 +I0916 17:25:17.479775 139652715497216 logging_writer.py:48] [135500] global_step=135500, grad_norm=0.344115, loss=1.38628 +I0916 17:25:17.483148 139681154073792 submission.py:307] 135500) loss = 1.386, grad_norm = 0.344 +I0916 17:26:36.223324 139652707104512 logging_writer.py:48] [136000] global_step=136000, grad_norm=0.35746, loss=1.39647 +I0916 17:26:36.226780 139681154073792 submission.py:307] 136000) loss = 1.396, grad_norm = 0.357 +I0916 17:27:55.069734 139652715497216 logging_writer.py:48] [136500] global_step=136500, grad_norm=0.365782, loss=1.3964 +I0916 17:27:55.073166 139681154073792 submission.py:307] 136500) loss = 1.396, grad_norm = 0.366 +I0916 17:29:13.858211 139652707104512 logging_writer.py:48] [137000] global_step=137000, grad_norm=0.376697, loss=1.4155 +I0916 17:29:13.861546 139681154073792 submission.py:307] 137000) loss = 1.416, grad_norm = 0.377 +I0916 17:30:32.745068 139652715497216 logging_writer.py:48] [137500] global_step=137500, grad_norm=0.34525, loss=1.37294 +I0916 17:30:32.748444 139681154073792 submission.py:307] 137500) loss = 1.373, grad_norm = 0.345 +I0916 17:31:51.710192 139652707104512 logging_writer.py:48] [138000] global_step=138000, grad_norm=0.346, loss=1.40924 +I0916 17:31:51.713304 139681154073792 submission.py:307] 138000) loss = 1.409, grad_norm = 0.346 +I0916 17:33:10.701521 139652715497216 logging_writer.py:48] [138500] global_step=138500, grad_norm=0.355587, loss=1.44081 +I0916 17:33:10.705086 139681154073792 submission.py:307] 138500) loss = 1.441, grad_norm = 0.356 +I0916 17:33:40.212823 139681154073792 spec.py:333] Evaluating on the training split. +I0916 17:33:42.444649 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 17:35:15.008208 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 17:35:17.221064 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 17:36:25.884498 139681154073792 spec.py:363] Evaluating on the test split. +I0916 17:36:28.098200 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 17:37:22.921287 139681154073792 submission_runner.py:516] Time since start: 31192.39s, Step: 138684, {'train/accuracy': 0.7082950367647058, 'train/loss': 1.2883000990923714, 'train/bleu': 36.50281435992577, 'validation/accuracy': 0.6930230251329804, 'validation/loss': 1.385253767002269, 'validation/bleu': 30.805347272254515, 'validation/num_examples': 3000, 'test/accuracy': 0.7096275637673581, 'test/loss': 1.2750479344605194, 'test/bleu': 30.415895458234928, 'test/num_examples': 3003, 'score': 21882.565131902695, 'total_duration': 31192.38531279564, 'accumulated_submission_time': 21882.565131902695, 'accumulated_eval_time': 9220.341248989105, 'accumulated_logging_time': 1.3608474731445312} +I0916 17:37:22.957202 139652707104512 logging_writer.py:48] [138684] accumulated_eval_time=9220.34, accumulated_logging_time=1.36085, accumulated_submission_time=21882.6, global_step=138684, preemption_count=0, score=21882.6, test/accuracy=0.709628, test/bleu=30.4159, test/loss=1.27505, test/num_examples=3003, total_duration=31192.4, train/accuracy=0.708295, train/bleu=36.5028, train/loss=1.2883, validation/accuracy=0.693023, validation/bleu=30.8053, validation/loss=1.38525, validation/num_examples=3000 +I0916 17:38:13.443780 139652715497216 logging_writer.py:48] [139000] global_step=139000, grad_norm=0.358333, loss=1.3653 +I0916 17:38:13.446822 139681154073792 submission.py:307] 139000) loss = 1.365, grad_norm = 0.358 +I0916 17:39:32.226182 139652707104512 logging_writer.py:48] [139500] global_step=139500, grad_norm=0.332911, loss=1.33374 +I0916 17:39:32.229685 139681154073792 submission.py:307] 139500) loss = 1.334, grad_norm = 0.333 +I0916 17:40:51.198785 139652715497216 logging_writer.py:48] [140000] global_step=140000, grad_norm=0.351484, loss=1.45202 +I0916 17:40:51.202069 139681154073792 submission.py:307] 140000) loss = 1.452, grad_norm = 0.351 +I0916 17:42:10.212661 139652707104512 logging_writer.py:48] [140500] global_step=140500, grad_norm=0.346696, loss=1.42942 +I0916 17:42:10.216051 139681154073792 submission.py:307] 140500) loss = 1.429, grad_norm = 0.347 +I0916 17:43:29.203241 139652715497216 logging_writer.py:48] [141000] global_step=141000, grad_norm=0.372118, loss=1.42608 +I0916 17:43:29.206897 139681154073792 submission.py:307] 141000) loss = 1.426, grad_norm = 0.372 +I0916 17:44:48.172295 139652707104512 logging_writer.py:48] [141500] global_step=141500, grad_norm=0.37343, loss=1.43383 +I0916 17:44:48.175595 139681154073792 submission.py:307] 141500) loss = 1.434, grad_norm = 0.373 +I0916 17:46:07.179160 139652715497216 logging_writer.py:48] [142000] global_step=142000, grad_norm=0.363017, loss=1.43877 +I0916 17:46:07.182380 139681154073792 submission.py:307] 142000) loss = 1.439, grad_norm = 0.363 +I0916 17:47:26.159076 139652707104512 logging_writer.py:48] [142500] global_step=142500, grad_norm=0.319529, loss=1.39945 +I0916 17:47:26.162426 139681154073792 submission.py:307] 142500) loss = 1.399, grad_norm = 0.320 +I0916 17:48:07.642058 139681154073792 spec.py:333] Evaluating on the training split. +I0916 17:48:09.875660 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 17:49:50.921936 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 17:49:53.136869 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 17:51:04.311860 139681154073792 spec.py:363] Evaluating on the test split. +I0916 17:51:06.523339 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 17:52:11.141732 139681154073792 submission_runner.py:516] Time since start: 32080.61s, Step: 142760, {'train/accuracy': 0.7007713844923147, 'train/loss': 1.3295150991371922, 'train/bleu': 36.41966519680081, 'validation/accuracy': 0.6934693928159601, 'validation/loss': 1.3867799221336374, 'validation/bleu': 30.73410874423248, 'validation/num_examples': 3000, 'test/accuracy': 0.709174365231538, 'test/loss': 1.2770021715763176, 'test/bleu': 30.39403426658349, 'test/num_examples': 3003, 'score': 22524.687610387802, 'total_duration': 32080.60576915741, 'accumulated_submission_time': 22524.687610387802, 'accumulated_eval_time': 9463.840975999832, 'accumulated_logging_time': 1.406548023223877} +I0916 17:52:11.174865 139652715497216 logging_writer.py:48] [142760] accumulated_eval_time=9463.84, accumulated_logging_time=1.40655, accumulated_submission_time=22524.7, global_step=142760, preemption_count=0, score=22524.7, test/accuracy=0.709174, test/bleu=30.394, test/loss=1.277, test/num_examples=3003, total_duration=32080.6, train/accuracy=0.700771, train/bleu=36.4197, train/loss=1.32952, validation/accuracy=0.693469, validation/bleu=30.7341, validation/loss=1.38678, validation/num_examples=3000 +I0916 17:52:49.635968 139652707104512 logging_writer.py:48] [143000] global_step=143000, grad_norm=0.37263, loss=1.43345 +I0916 17:52:49.639466 139681154073792 submission.py:307] 143000) loss = 1.433, grad_norm = 0.373 +I0916 17:54:08.392177 139652715497216 logging_writer.py:48] [143500] global_step=143500, grad_norm=0.365825, loss=1.40144 +I0916 17:54:08.395586 139681154073792 submission.py:307] 143500) loss = 1.401, grad_norm = 0.366 +I0916 17:55:27.290470 139652707104512 logging_writer.py:48] [144000] global_step=144000, grad_norm=0.326799, loss=1.41544 +I0916 17:55:27.294051 139681154073792 submission.py:307] 144000) loss = 1.415, grad_norm = 0.327 +I0916 17:56:46.112848 139652715497216 logging_writer.py:48] [144500] global_step=144500, grad_norm=0.377954, loss=1.44644 +I0916 17:56:46.116278 139681154073792 submission.py:307] 144500) loss = 1.446, grad_norm = 0.378 +I0916 17:58:04.998878 139652707104512 logging_writer.py:48] [145000] global_step=145000, grad_norm=0.363196, loss=1.46676 +I0916 17:58:05.002482 139681154073792 submission.py:307] 145000) loss = 1.467, grad_norm = 0.363 +I0916 17:59:23.839323 139652715497216 logging_writer.py:48] [145500] global_step=145500, grad_norm=0.347937, loss=1.41562 +I0916 17:59:23.842732 139681154073792 submission.py:307] 145500) loss = 1.416, grad_norm = 0.348 +I0916 18:00:42.691249 139652707104512 logging_writer.py:48] [146000] global_step=146000, grad_norm=0.394909, loss=1.43483 +I0916 18:00:42.694554 139681154073792 submission.py:307] 146000) loss = 1.435, grad_norm = 0.395 +I0916 18:02:01.647889 139652715497216 logging_writer.py:48] [146500] global_step=146500, grad_norm=0.363211, loss=1.38452 +I0916 18:02:01.651383 139681154073792 submission.py:307] 146500) loss = 1.385, grad_norm = 0.363 +I0916 18:02:55.721954 139681154073792 spec.py:333] Evaluating on the training split. +I0916 18:02:57.953169 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 18:04:46.494888 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 18:04:48.709625 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 18:06:07.318115 139681154073792 spec.py:363] Evaluating on the test split. +I0916 18:06:09.536398 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 18:07:14.165809 139681154073792 submission_runner.py:516] Time since start: 32983.63s, Step: 146840, {'train/accuracy': 0.7025382336456082, 'train/loss': 1.3191507633860655, 'train/bleu': 36.45069613420243, 'validation/accuracy': 0.6923782718131207, 'validation/loss': 1.3888317798291403, 'validation/bleu': 30.375924389895964, 'validation/num_examples': 3000, 'test/accuracy': 0.7088489919237697, 'test/loss': 1.2799526828772296, 'test/bleu': 30.26458936712677, 'test/num_examples': 3003, 'score': 23166.677849292755, 'total_duration': 32983.62984442711, 'accumulated_submission_time': 23166.677849292755, 'accumulated_eval_time': 9722.284867286682, 'accumulated_logging_time': 1.4494123458862305} +I0916 18:07:14.198175 139652707104512 logging_writer.py:48] [146840] accumulated_eval_time=9722.28, accumulated_logging_time=1.44941, accumulated_submission_time=23166.7, global_step=146840, preemption_count=0, score=23166.7, test/accuracy=0.708849, test/bleu=30.2646, test/loss=1.27995, test/num_examples=3003, total_duration=32983.6, train/accuracy=0.702538, train/bleu=36.4507, train/loss=1.31915, validation/accuracy=0.692378, validation/bleu=30.3759, validation/loss=1.38883, validation/num_examples=3000 +I0916 18:07:40.136319 139652715497216 logging_writer.py:48] [147000] global_step=147000, grad_norm=0.350209, loss=1.42653 +I0916 18:07:40.139575 139681154073792 submission.py:307] 147000) loss = 1.427, grad_norm = 0.350 +I0916 18:08:58.888790 139652707104512 logging_writer.py:48] [147500] global_step=147500, grad_norm=0.339527, loss=1.42534 +I0916 18:08:58.892009 139681154073792 submission.py:307] 147500) loss = 1.425, grad_norm = 0.340 +I0916 18:10:17.662200 139652715497216 logging_writer.py:48] [148000] global_step=148000, grad_norm=0.349417, loss=1.41615 +I0916 18:10:17.665629 139681154073792 submission.py:307] 148000) loss = 1.416, grad_norm = 0.349 +I0916 18:11:36.538991 139652707104512 logging_writer.py:48] [148500] global_step=148500, grad_norm=0.383673, loss=1.55377 +I0916 18:11:36.542486 139681154073792 submission.py:307] 148500) loss = 1.554, grad_norm = 0.384 +I0916 18:12:55.434178 139652715497216 logging_writer.py:48] [149000] global_step=149000, grad_norm=0.354408, loss=1.43185 +I0916 18:12:55.437500 139681154073792 submission.py:307] 149000) loss = 1.432, grad_norm = 0.354 +I0916 18:14:14.321395 139652707104512 logging_writer.py:48] [149500] global_step=149500, grad_norm=0.351704, loss=1.42775 +I0916 18:14:14.324962 139681154073792 submission.py:307] 149500) loss = 1.428, grad_norm = 0.352 +I0916 18:15:33.200917 139652715497216 logging_writer.py:48] [150000] global_step=150000, grad_norm=0.349827, loss=1.38032 +I0916 18:15:33.204134 139681154073792 submission.py:307] 150000) loss = 1.380, grad_norm = 0.350 +I0916 18:16:52.171804 139652707104512 logging_writer.py:48] [150500] global_step=150500, grad_norm=0.395615, loss=1.41919 +I0916 18:16:52.175174 139681154073792 submission.py:307] 150500) loss = 1.419, grad_norm = 0.396 +I0916 18:17:58.862005 139681154073792 spec.py:333] Evaluating on the training split. +I0916 18:18:01.093108 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 18:19:40.460443 139681154073792 spec.py:346] Evaluating on the validation split. +I0916 18:19:42.681030 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 18:20:59.736371 139681154073792 spec.py:363] Evaluating on the test split. +I0916 18:21:01.955474 139681154073792 workload.py:152] Translating evaluation dataset. +I0916 18:22:15.593895 139681154073792 submission_runner.py:516] Time since start: 33885.06s, Step: 150920, {'train/accuracy': 0.7043234266735383, 'train/loss': 1.307740741138407, 'train/bleu': 36.57719162324027, 'validation/accuracy': 0.6922666798923758, 'validation/loss': 1.3949919057110265, 'validation/bleu': 30.715556801482006, 'validation/num_examples': 3000, 'test/accuracy': 0.7084887571901691, 'test/loss': 1.286095738190692, 'test/bleu': 30.23243484427275, 'test/num_examples': 3003, 'score': 23808.78427028656, 'total_duration': 33885.05792140961, 'accumulated_submission_time': 23808.78427028656, 'accumulated_eval_time': 9979.016785621643, 'accumulated_logging_time': 1.4916207790374756} +I0916 18:22:15.627197 139652715497216 logging_writer.py:48] [150920] accumulated_eval_time=9979.02, accumulated_logging_time=1.49162, accumulated_submission_time=23808.8, global_step=150920, preemption_count=0, score=23808.8, test/accuracy=0.708489, test/bleu=30.2324, test/loss=1.2861, test/num_examples=3003, total_duration=33885.1, train/accuracy=0.704323, train/bleu=36.5772, train/loss=1.30774, validation/accuracy=0.692267, validation/bleu=30.7156, validation/loss=1.39499, validation/num_examples=3000 +I0916 18:22:28.983714 139652707104512 logging_writer.py:48] [151000] global_step=151000, grad_norm=0.368661, loss=1.40979 +I0916 18:22:28.986950 139681154073792 submission.py:307] 151000) loss = 1.410, grad_norm = 0.369 +I0916 18:23:47.554169 139652715497216 logging_writer.py:48] [151500] global_step=151500, grad_norm=0.362175, loss=1.43097 +I0916 18:23:47.557625 139681154073792 submission.py:307] 151500) loss = 1.431, grad_norm = 0.362 +I0916 18:25:06.403636 139652707104512 logging_writer.py:48] [152000] global_step=152000, grad_norm=0.363246, loss=1.38979 +I0916 18:25:06.406957 139681154073792 submission.py:307] 152000) loss = 1.390, grad_norm = 0.363 +I0916 18:26:25.239039 139652715497216 logging_writer.py:48] [152500] global_step=152500, grad_norm=0.372647, loss=1.51089 +I0916 18:26:25.242548 139681154073792 submission.py:307] 152500) loss = 1.511, grad_norm = 0.373 +I0916 18:27:44.084573 139652707104512 logging_writer.py:48] [153000] global_step=153000, grad_norm=0.359226, loss=1.46022 +I0916 18:27:44.087918 139681154073792 submission.py:307] 153000) loss = 1.460, grad_norm = 0.359 +I0916 18:29:02.956538 139652715497216 logging_writer.py:48] [153500] global_step=153500, grad_norm=0.386915, loss=1.43888 +I0916 18:29:02.959857 139681154073792 submission.py:307] 153500) loss = 1.439, grad_norm = 0.387 +I0916 18:30:21.810527 139652707104512 logging_writer.py:48] [154000] global_step=154000, grad_norm=0.385597, loss=1.45302 +I0916 18:30:21.813726 139681154073792 submission.py:307] 154000) loss = 1.453, grad_norm = 0.386 +I0916 18:31:40.721098 139652715497216 logging_writer.py:48] [154500] global_step=154500, grad_norm=0.386805, loss=1.46684 +I0916 18:31:40.724568 139681154073792 submission.py:307] 154500) loss = 1.467, grad_norm = 0.387 +I0916 18:32:59.621762 139652707104512 logging_writer.py:48] [155000] global_step=155000, grad_norm=0.381962, loss=1.42757 +I0916 18:32:59.625609 139681154073792 submission.py:307] 155000) loss = 1.428, grad_norm = 0.382 +I0916 18:32:59.643557 139652715497216 logging_writer.py:48] [155001] global_step=155001, preemption_count=0, score=24450.8 +I0916 18:32:59.662120 139681154073792 submission_runner.py:857] Final wmt score: 24450.802146673203 diff --git a/logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/criteo1tb_pytorch_09-14-2026-22-26-15.log b/logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/criteo1tb_pytorch_09-14-2026-22-26-15.log new file mode 100644 index 00000000..ffaf298c --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/criteo1tb_pytorch_09-14-2026-22-26-15.log @@ -0,0 +1,523 @@ +torchrun --redirects 1:0,2:0,3:0 --standalone --nnodes=1 --nproc_per_node=4 submission_runner.py --framework=pytorch --workload=criteo1tb --submission_path=submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py --data_dir=/data/criteo1tb --experiment_dir=/experiment_runs --experiment_name=submissions_a100/ademamix_golden/study_2 --overwrite=True --save_checkpoints=False --rng_seed=-826981189 --torch_compile=true --tuning_ruleset=self 2>&1 | tee -a /logs/criteo1tb_pytorch_09-14-2026-22-26-15.log +W0914 22:26:43.657000 9 site-packages/torch/distributed/run.py:803] +W0914 22:26:43.657000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0914 22:26:43.657000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0914 22:26:43.657000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-14 22:26:59.801131: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-14 22:26:59.801131: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-14 22:26:59.801176: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-14 22:26:59.801131: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789424820.321323 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789424820.321311 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789424820.321333 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789424820.321304 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789424820.406534 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789424820.406539 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789424820.406543 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789424820.406560 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789424821.741981 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789424821.741981 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789424821.741983 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789424821.741985 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789424821.742024 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789424821.742024 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789424821.742024 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789424821.742027 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789424821.742026 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789424821.742027 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789424821.742028 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789424821.742029 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789424821.742030 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789424821.742030 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789424821.742031 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789424821.742032 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789424852.251685 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789424852.251717 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789424852.396547 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789424852.410173 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W914 22:27:40.445159598 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0914 22:27:43.228543 140430692230336 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/criteo1tb_pytorch. +I0914 22:27:43.228545 139969034814656 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/criteo1tb_pytorch. +I0914 22:27:43.228534 140006173705408 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/criteo1tb_pytorch. +I0914 22:27:43.228574 139983003481280 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/criteo1tb_pytorch. +I0914 22:27:43.455733 140430692230336 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/criteo1tb_pytorch/trial_1. +I0914 22:27:43.728600 140430692230336 submission_runner.py:242] Initializing dataset. +I0914 22:27:43.728780 140430692230336 submission_runner.py:251] Initializing model. +W0914 22:27:58.932130 140006173705408 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0914 22:27:58.932133 140430692230336 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0914 22:27:58.932135 139983003481280 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0914 22:27:58.932179 139969034814656 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +I0914 22:27:58.932327 140430692230336 submission_runner.py:294] Initializing optimizer. +I0914 22:27:58.932711 140430692230336 submission_runner.py:299] Initializing metrics bundle. +I0914 22:27:58.932871 140430692230336 submission_runner.py:321] Initializing checkpoint and logger. +I0914 22:27:58.934438 140430692230336 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_2/criteo1tb_pytorch/trial_1/meta_data_0.json. +I0914 22:27:58.934554 140006173705408 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0914 22:27:58.934573 139983003481280 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0914 22:27:58.934639 140430692230336 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0914 22:27:58.934574 139969034814656 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0914 22:27:58.934685 139983003481280 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0914 22:27:58.934686 140006173705408 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0914 22:27:58.934695 140430692230336 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0914 22:27:58.934710 139969034814656 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0914 22:27:59.354190 140430692230336 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_2/criteo1tb_pytorch/trial_1/flags_0.json. +I0914 22:27:59.481465 140430692230336 submission_runner.py:359] Starting training loop. +I0914 22:28:10.453137 140406406772480 logging_writer.py:48] [0] global_step=0, grad_norm=0.5, loss=1.9295 +I0914 22:28:10.706327 140430692230336 submission.py:307] 0) loss = 1.929, grad_norm = 0.500 +I0914 22:28:11.029697 140430692230336 spec.py:333] Evaluating on the training split. +I0914 22:39:03.003396 140430692230336 spec.py:346] Evaluating on the validation split. +I0914 22:50:14.415499 140430692230336 spec.py:363] Evaluating on the test split. +I0914 23:02:43.954076 140430692230336 submission_runner.py:516] Time since start: 2084.47s, Step: 1, {'train/loss': 1.9395356163303379, 'validation/loss': 1.9414668747834611, 'validation/num_examples': 83274637, 'test/loss': 1.9351263324861225, 'test/num_examples': 95000000, 'score': 11.226190328598022, 'total_duration': 2084.472838163376, 'accumulated_submission_time': 11.226190328598022, 'accumulated_eval_time': 2072.9244389533997, 'accumulated_logging_time': 0} +I0914 23:02:44.041604 140387398096640 logging_writer.py:48] [1] accumulated_eval_time=2072.92, accumulated_logging_time=0, accumulated_submission_time=11.2262, global_step=1, preemption_count=0, score=11.2262, test/loss=1.93513, test/num_examples=95000000, total_duration=2084.47, train/loss=1.93954, validation/loss=1.94147, validation/num_examples=83274637 +I0914 23:02:44.634194 140387389703936 logging_writer.py:48] [1] global_step=1, grad_norm=0.5, loss=1.92905 +I0914 23:02:44.637041 140430692230336 submission.py:307] 1) loss = 1.929, grad_norm = 0.500 +I0914 23:02:44.871863 140387398096640 logging_writer.py:48] [2] global_step=2, grad_norm=0.5, loss=1.92104 +I0914 23:02:44.874739 140430692230336 submission.py:307] 2) loss = 1.921, grad_norm = 0.500 +I0914 23:02:45.106596 140387389703936 logging_writer.py:48] [3] global_step=3, grad_norm=0.5, loss=1.90505 +I0914 23:02:45.109170 140430692230336 submission.py:307] 3) loss = 1.905, grad_norm = 0.500 +I0914 23:02:45.343038 140387398096640 logging_writer.py:48] [4] global_step=4, grad_norm=0.5, loss=1.88017 +I0914 23:02:45.345636 140430692230336 submission.py:307] 4) loss = 1.880, grad_norm = 0.500 +I0914 23:02:45.578295 140387389703936 logging_writer.py:48] [5] global_step=5, grad_norm=0.5, loss=1.84612 +I0914 23:02:45.580843 140430692230336 submission.py:307] 5) loss = 1.846, grad_norm = 0.500 +I0914 23:02:45.812871 140387398096640 logging_writer.py:48] [6] global_step=6, grad_norm=0.5, loss=1.80347 +I0914 23:02:45.815384 140430692230336 submission.py:307] 6) loss = 1.803, grad_norm = 0.500 +I0914 23:02:46.048357 140387389703936 logging_writer.py:48] [7] global_step=7, grad_norm=0.5, loss=1.75278 +I0914 23:02:46.050935 140430692230336 submission.py:307] 7) loss = 1.753, grad_norm = 0.500 +I0914 23:02:46.285104 140387398096640 logging_writer.py:48] [8] global_step=8, grad_norm=0.5, loss=1.69302 +I0914 23:02:46.287961 140430692230336 submission.py:307] 8) loss = 1.693, grad_norm = 0.500 +I0914 23:02:46.522460 140387389703936 logging_writer.py:48] [9] global_step=9, grad_norm=0.5, loss=1.62577 +I0914 23:02:46.525203 140430692230336 submission.py:307] 9) loss = 1.626, grad_norm = 0.500 +I0914 23:02:46.759504 140387398096640 logging_writer.py:48] [10] global_step=10, grad_norm=0.5, loss=1.54895 +I0914 23:02:46.762338 140430692230336 submission.py:307] 10) loss = 1.549, grad_norm = 0.500 +I0914 23:02:46.994683 140387389703936 logging_writer.py:48] [11] global_step=11, grad_norm=0.5, loss=1.46508 +I0914 23:02:46.997619 140430692230336 submission.py:307] 11) loss = 1.465, grad_norm = 0.500 +I0914 23:02:47.232993 140387398096640 logging_writer.py:48] [12] global_step=12, grad_norm=0.5, loss=1.37202 +I0914 23:02:47.235915 140430692230336 submission.py:307] 12) loss = 1.372, grad_norm = 0.500 +I0914 23:02:47.469985 140387389703936 logging_writer.py:48] [13] global_step=13, grad_norm=0.5, loss=1.27374 +I0914 23:02:47.472707 140430692230336 submission.py:307] 13) loss = 1.274, grad_norm = 0.500 +I0914 23:02:47.702679 140387398096640 logging_writer.py:48] [14] global_step=14, grad_norm=0.5, loss=1.16958 +I0914 23:02:47.705397 140430692230336 submission.py:307] 14) loss = 1.170, grad_norm = 0.500 +I0914 23:02:47.936929 140387389703936 logging_writer.py:48] [15] global_step=15, grad_norm=0.5, loss=1.0591 +I0914 23:02:47.939608 140430692230336 submission.py:307] 15) loss = 1.059, grad_norm = 0.500 +I0914 23:02:48.171444 140387398096640 logging_writer.py:48] [16] global_step=16, grad_norm=0.5, loss=0.947731 +I0914 23:02:48.174058 140430692230336 submission.py:307] 16) loss = 0.948, grad_norm = 0.500 +I0914 23:02:48.407999 140387389703936 logging_writer.py:48] [17] global_step=17, grad_norm=0.5, loss=0.834504 +I0914 23:02:48.410865 140430692230336 submission.py:307] 17) loss = 0.835, grad_norm = 0.500 +I0914 23:02:48.646030 140387398096640 logging_writer.py:48] [18] global_step=18, grad_norm=0.5, loss=0.72431 +I0914 23:02:48.648757 140430692230336 submission.py:307] 18) loss = 0.724, grad_norm = 0.500 +I0914 23:02:48.881209 140387389703936 logging_writer.py:48] [19] global_step=19, grad_norm=0.5, loss=0.636142 +I0914 23:02:48.883986 140430692230336 submission.py:307] 19) loss = 0.636, grad_norm = 0.500 +I0914 23:02:49.116657 140387398096640 logging_writer.py:48] [20] global_step=20, grad_norm=0.5, loss=0.540781 +I0914 23:02:49.119635 140430692230336 submission.py:307] 20) loss = 0.541, grad_norm = 0.500 +I0914 23:02:49.353076 140387389703936 logging_writer.py:48] [21] global_step=21, grad_norm=0.5, loss=0.454188 +I0914 23:02:49.356092 140430692230336 submission.py:307] 21) loss = 0.454, grad_norm = 0.500 +I0914 23:02:49.589178 140387398096640 logging_writer.py:48] [22] global_step=22, grad_norm=0.5, loss=0.383834 +I0914 23:02:49.591818 140430692230336 submission.py:307] 22) loss = 0.384, grad_norm = 0.500 +I0914 23:02:49.823233 140387389703936 logging_writer.py:48] [23] global_step=23, grad_norm=0.5, loss=0.325052 +I0914 23:02:49.825895 140430692230336 submission.py:307] 23) loss = 0.325, grad_norm = 0.500 +I0914 23:02:50.058040 140387398096640 logging_writer.py:48] [24] global_step=24, grad_norm=0.5, loss=0.281876 +I0914 23:02:50.060604 140430692230336 submission.py:307] 24) loss = 0.282, grad_norm = 0.500 +I0914 23:02:50.292833 140387389703936 logging_writer.py:48] [25] global_step=25, grad_norm=0.5, loss=0.250862 +I0914 23:02:50.295437 140430692230336 submission.py:307] 25) loss = 0.251, grad_norm = 0.500 +I0914 23:02:50.527888 140387398096640 logging_writer.py:48] [26] global_step=26, grad_norm=0.499999, loss=0.234136 +I0914 23:02:50.530688 140430692230336 submission.py:307] 26) loss = 0.234, grad_norm = 0.500 +I0914 23:02:50.762810 140387389703936 logging_writer.py:48] [27] global_step=27, grad_norm=0.39336, loss=0.223328 +I0914 23:02:50.765589 140430692230336 submission.py:307] 27) loss = 0.223, grad_norm = 0.393 +I0914 23:02:50.997238 140387398096640 logging_writer.py:48] [28] global_step=28, grad_norm=0.499999, loss=0.226137 +I0914 23:02:50.999999 140430692230336 submission.py:307] 28) loss = 0.226, grad_norm = 0.500 +I0914 23:02:51.938717 140387389703936 logging_writer.py:48] [29] global_step=29, grad_norm=0.499999, loss=0.222923 +I0914 23:02:51.941550 140430692230336 submission.py:307] 29) loss = 0.223, grad_norm = 0.500 +I0914 23:02:54.028677 140387398096640 logging_writer.py:48] [30] global_step=30, grad_norm=0.499999, loss=0.223002 +I0914 23:02:54.031557 140430692230336 submission.py:307] 30) loss = 0.223, grad_norm = 0.500 +I0914 23:02:56.130604 140387389703936 logging_writer.py:48] [31] global_step=31, grad_norm=0.499999, loss=0.224685 +I0914 23:02:56.133478 140430692230336 submission.py:307] 31) loss = 0.225, grad_norm = 0.500 +I0914 23:02:58.240663 140387398096640 logging_writer.py:48] [32] global_step=32, grad_norm=0.499999, loss=0.217748 +I0914 23:02:58.243553 140430692230336 submission.py:307] 32) loss = 0.218, grad_norm = 0.500 +I0914 23:02:59.952702 140387389703936 logging_writer.py:48] [33] global_step=33, grad_norm=0.499999, loss=0.210751 +I0914 23:02:59.955524 140430692230336 submission.py:307] 33) loss = 0.211, grad_norm = 0.500 +I0914 23:03:02.232655 140387398096640 logging_writer.py:48] [34] global_step=34, grad_norm=0.366001, loss=0.20293 +I0914 23:03:02.235592 140430692230336 submission.py:307] 34) loss = 0.203, grad_norm = 0.366 +I0914 23:03:04.489333 140387389703936 logging_writer.py:48] [35] global_step=35, grad_norm=0.279079, loss=0.199795 +I0914 23:03:04.492147 140430692230336 submission.py:307] 35) loss = 0.200, grad_norm = 0.279 +I0914 23:03:06.614038 140387398096640 logging_writer.py:48] [36] global_step=36, grad_norm=0.411395, loss=0.195405 +I0914 23:03:06.616852 140430692230336 submission.py:307] 36) loss = 0.195, grad_norm = 0.411 +I0914 23:03:08.927439 140387389703936 logging_writer.py:48] [37] global_step=37, grad_norm=0.418858, loss=0.191518 +I0914 23:03:08.930265 140430692230336 submission.py:307] 37) loss = 0.192, grad_norm = 0.419 +I0914 23:03:10.932263 140387398096640 logging_writer.py:48] [38] global_step=38, grad_norm=0.240729, loss=0.186488 +I0914 23:03:10.935114 140430692230336 submission.py:307] 38) loss = 0.186, grad_norm = 0.241 +I0914 23:03:12.942209 140387389703936 logging_writer.py:48] [39] global_step=39, grad_norm=0.236696, loss=0.181854 +I0914 23:03:12.945029 140430692230336 submission.py:307] 39) loss = 0.182, grad_norm = 0.237 +I0914 23:03:15.073035 140387398096640 logging_writer.py:48] [40] global_step=40, grad_norm=0.327448, loss=0.175272 +I0914 23:03:15.075908 140430692230336 submission.py:307] 40) loss = 0.175, grad_norm = 0.327 +I0914 23:03:17.046324 140387389703936 logging_writer.py:48] [41] global_step=41, grad_norm=0.416997, loss=0.178122 +I0914 23:03:17.049175 140430692230336 submission.py:307] 41) loss = 0.178, grad_norm = 0.417 +I0914 23:03:19.063582 140387398096640 logging_writer.py:48] [42] global_step=42, grad_norm=0.251776, loss=0.17229 +I0914 23:03:19.066422 140430692230336 submission.py:307] 42) loss = 0.172, grad_norm = 0.252 +I0914 23:03:21.218919 140387389703936 logging_writer.py:48] [43] global_step=43, grad_norm=0.142066, loss=0.168994 +I0914 23:03:21.221781 140430692230336 submission.py:307] 43) loss = 0.169, grad_norm = 0.142 +I0914 23:03:23.402291 140387398096640 logging_writer.py:48] [44] global_step=44, grad_norm=0.242732, loss=0.166525 +I0914 23:03:23.405148 140430692230336 submission.py:307] 44) loss = 0.167, grad_norm = 0.243 +I0914 23:03:25.752980 140387389703936 logging_writer.py:48] [45] global_step=45, grad_norm=0.180022, loss=0.163673 +I0914 23:03:25.755795 140430692230336 submission.py:307] 45) loss = 0.164, grad_norm = 0.180 +I0914 23:03:27.802547 140387398096640 logging_writer.py:48] [46] global_step=46, grad_norm=0.14188, loss=0.16333 +I0914 23:03:27.805367 140430692230336 submission.py:307] 46) loss = 0.163, grad_norm = 0.142 +I0914 23:03:30.100560 140387389703936 logging_writer.py:48] [47] global_step=47, grad_norm=0.199593, loss=0.160264 +I0914 23:03:30.103356 140430692230336 submission.py:307] 47) loss = 0.160, grad_norm = 0.200 +I0914 23:03:32.579326 140387398096640 logging_writer.py:48] [48] global_step=48, grad_norm=0.123993, loss=0.153118 +I0914 23:03:32.582230 140430692230336 submission.py:307] 48) loss = 0.153, grad_norm = 0.124 +I0914 23:03:34.883906 140387389703936 logging_writer.py:48] [49] global_step=49, grad_norm=0.106184, loss=0.152692 +I0914 23:03:34.886793 140430692230336 submission.py:307] 49) loss = 0.153, grad_norm = 0.106 +I0914 23:03:36.873198 140387398096640 logging_writer.py:48] [50] global_step=50, grad_norm=0.104461, loss=0.148599 +I0914 23:03:36.876121 140430692230336 submission.py:307] 50) loss = 0.149, grad_norm = 0.104 +I0914 23:03:39.034761 140387389703936 logging_writer.py:48] [51] global_step=51, grad_norm=0.14195, loss=0.147986 +I0914 23:03:39.037581 140430692230336 submission.py:307] 51) loss = 0.148, grad_norm = 0.142 +I0914 23:03:41.113715 140387398096640 logging_writer.py:48] [52] global_step=52, grad_norm=0.0898748, loss=0.143886 +I0914 23:03:41.116516 140430692230336 submission.py:307] 52) loss = 0.144, grad_norm = 0.090 +I0914 23:03:43.035330 140387389703936 logging_writer.py:48] [53] global_step=53, grad_norm=0.0972307, loss=0.141108 +I0914 23:03:43.038150 140430692230336 submission.py:307] 53) loss = 0.141, grad_norm = 0.097 +I0914 23:03:45.350298 140387398096640 logging_writer.py:48] [54] global_step=54, grad_norm=0.0722353, loss=0.140235 +I0914 23:03:45.353144 140430692230336 submission.py:307] 54) loss = 0.140, grad_norm = 0.072 +I0914 23:03:47.656296 140387389703936 logging_writer.py:48] [55] global_step=55, grad_norm=0.106609, loss=0.139295 +I0914 23:03:47.659134 140430692230336 submission.py:307] 55) loss = 0.139, grad_norm = 0.107 +I0914 23:03:49.636959 140387398096640 logging_writer.py:48] [56] global_step=56, grad_norm=0.0651724, loss=0.138552 +I0914 23:03:49.639810 140430692230336 submission.py:307] 56) loss = 0.139, grad_norm = 0.065 +I0914 23:03:52.126024 140387389703936 logging_writer.py:48] [57] global_step=57, grad_norm=0.112882, loss=0.13618 +I0914 23:03:52.128804 140430692230336 submission.py:307] 57) loss = 0.136, grad_norm = 0.113 +I0914 23:03:54.237980 140387398096640 logging_writer.py:48] [58] global_step=58, grad_norm=0.101619, loss=0.134654 +I0914 23:03:54.240794 140430692230336 submission.py:307] 58) loss = 0.135, grad_norm = 0.102 +I0914 23:03:56.441434 140387389703936 logging_writer.py:48] [59] global_step=59, grad_norm=0.0567257, loss=0.133586 +I0914 23:03:56.444210 140430692230336 submission.py:307] 59) loss = 0.134, grad_norm = 0.057 +I0914 23:03:58.412776 140387398096640 logging_writer.py:48] [60] global_step=60, grad_norm=0.132492, loss=0.131982 +I0914 23:03:58.415731 140430692230336 submission.py:307] 60) loss = 0.132, grad_norm = 0.132 +I0914 23:04:00.465005 140387389703936 logging_writer.py:48] [61] global_step=61, grad_norm=0.216153, loss=0.134381 +I0914 23:04:00.467797 140430692230336 submission.py:307] 61) loss = 0.134, grad_norm = 0.216 +I0914 23:04:02.811802 140387398096640 logging_writer.py:48] [62] global_step=62, grad_norm=0.29776, loss=0.132264 +I0914 23:04:02.814677 140430692230336 submission.py:307] 62) loss = 0.132, grad_norm = 0.298 +I0914 23:04:04.923826 140387389703936 logging_writer.py:48] [63] global_step=63, grad_norm=0.318907, loss=0.132512 +I0914 23:04:04.926633 140430692230336 submission.py:307] 63) loss = 0.133, grad_norm = 0.319 +I0914 23:04:07.362839 140387398096640 logging_writer.py:48] [64] global_step=64, grad_norm=0.205518, loss=0.133203 +I0914 23:04:07.365694 140430692230336 submission.py:307] 64) loss = 0.133, grad_norm = 0.206 +I0914 23:04:09.422798 140387389703936 logging_writer.py:48] [65] global_step=65, grad_norm=0.03573, loss=0.132058 +I0914 23:04:09.425666 140430692230336 submission.py:307] 65) loss = 0.132, grad_norm = 0.036 +I0914 23:04:11.635332 140387398096640 logging_writer.py:48] [66] global_step=66, grad_norm=0.155827, loss=0.131264 +I0914 23:04:11.638214 140430692230336 submission.py:307] 66) loss = 0.131, grad_norm = 0.156 +I0914 23:04:13.725629 140387389703936 logging_writer.py:48] [67] global_step=67, grad_norm=0.199905, loss=0.134263 +I0914 23:04:13.728471 140430692230336 submission.py:307] 67) loss = 0.134, grad_norm = 0.200 +I0914 23:04:15.874047 140387398096640 logging_writer.py:48] [68] global_step=68, grad_norm=0.178979, loss=0.133268 +I0914 23:04:15.876847 140430692230336 submission.py:307] 68) loss = 0.133, grad_norm = 0.179 +I0914 23:04:18.115293 140387389703936 logging_writer.py:48] [69] global_step=69, grad_norm=0.143134, loss=0.132882 +I0914 23:04:18.118114 140430692230336 submission.py:307] 69) loss = 0.133, grad_norm = 0.143 +I0914 23:04:20.280470 140387398096640 logging_writer.py:48] [70] global_step=70, grad_norm=0.146267, loss=0.134626 +I0914 23:04:20.283284 140430692230336 submission.py:307] 70) loss = 0.135, grad_norm = 0.146 +I0914 23:04:22.404596 140387389703936 logging_writer.py:48] [71] global_step=71, grad_norm=0.247903, loss=0.13033 +I0914 23:04:22.407385 140430692230336 submission.py:307] 71) loss = 0.130, grad_norm = 0.248 +I0914 23:04:24.725711 140387398096640 logging_writer.py:48] [72] global_step=72, grad_norm=0.404345, loss=0.133258 +I0914 23:04:24.728502 140430692230336 submission.py:307] 72) loss = 0.133, grad_norm = 0.404 +I0914 23:04:26.906914 140387389703936 logging_writer.py:48] [73] global_step=73, grad_norm=0.499999, loss=0.133942 +I0914 23:04:26.909829 140430692230336 submission.py:307] 73) loss = 0.134, grad_norm = 0.500 +I0914 23:04:29.223294 140387398096640 logging_writer.py:48] [74] global_step=74, grad_norm=0.450488, loss=0.132875 +I0914 23:04:29.226192 140430692230336 submission.py:307] 74) loss = 0.133, grad_norm = 0.450 +I0914 23:04:31.106035 140387389703936 logging_writer.py:48] [75] global_step=75, grad_norm=0.185724, loss=0.131912 +I0914 23:04:31.108898 140430692230336 submission.py:307] 75) loss = 0.132, grad_norm = 0.186 +I0914 23:04:33.573101 140387398096640 logging_writer.py:48] [76] global_step=76, grad_norm=0.127402, loss=0.131882 +I0914 23:04:33.575940 140430692230336 submission.py:307] 76) loss = 0.132, grad_norm = 0.127 +I0914 23:04:35.976170 140387389703936 logging_writer.py:48] [77] global_step=77, grad_norm=0.33013, loss=0.132283 +I0914 23:04:35.979029 140430692230336 submission.py:307] 77) loss = 0.132, grad_norm = 0.330 +I0914 23:04:38.269359 140387398096640 logging_writer.py:48] [78] global_step=78, grad_norm=0.422593, loss=0.131522 +I0914 23:04:38.272680 140430692230336 submission.py:307] 78) loss = 0.132, grad_norm = 0.423 +I0914 23:04:40.520259 140387389703936 logging_writer.py:48] [79] global_step=79, grad_norm=0.414949, loss=0.13177 +I0914 23:04:40.523099 140430692230336 submission.py:307] 79) loss = 0.132, grad_norm = 0.415 +I0914 23:04:42.718775 140387398096640 logging_writer.py:48] [80] global_step=80, grad_norm=0.293137, loss=0.130834 +I0914 23:04:42.721562 140430692230336 submission.py:307] 80) loss = 0.131, grad_norm = 0.293 +I0914 23:04:44.945537 140387389703936 logging_writer.py:48] [81] global_step=81, grad_norm=0.139599, loss=0.131651 +I0914 23:04:44.948335 140430692230336 submission.py:307] 81) loss = 0.132, grad_norm = 0.140 +I0914 23:04:47.072991 140387398096640 logging_writer.py:48] [82] global_step=82, grad_norm=0.0212233, loss=0.129501 +I0914 23:04:47.075776 140430692230336 submission.py:307] 82) loss = 0.130, grad_norm = 0.021 +I0914 23:04:49.239839 140387389703936 logging_writer.py:48] [83] global_step=83, grad_norm=0.0839629, loss=0.129585 +I0914 23:04:49.242631 140430692230336 submission.py:307] 83) loss = 0.130, grad_norm = 0.084 +I0914 23:04:51.165052 140387398096640 logging_writer.py:48] [84] global_step=84, grad_norm=0.138993, loss=0.129141 +I0914 23:04:51.167833 140430692230336 submission.py:307] 84) loss = 0.129, grad_norm = 0.139 +I0914 23:04:53.398364 140387389703936 logging_writer.py:48] [85] global_step=85, grad_norm=0.158664, loss=0.128606 +I0914 23:04:53.401169 140430692230336 submission.py:307] 85) loss = 0.129, grad_norm = 0.159 +I0914 23:04:55.742145 140387398096640 logging_writer.py:48] [86] global_step=86, grad_norm=0.263354, loss=0.131454 +I0914 23:04:55.744899 140430692230336 submission.py:307] 86) loss = 0.131, grad_norm = 0.263 +I0914 23:04:58.017858 140387389703936 logging_writer.py:48] [87] global_step=87, grad_norm=0.499999, loss=0.13145 +I0914 23:04:58.020595 140430692230336 submission.py:307] 87) loss = 0.131, grad_norm = 0.500 +I0914 23:05:00.319971 140387398096640 logging_writer.py:48] [88] global_step=88, grad_norm=0.499999, loss=0.131742 +I0914 23:05:00.322756 140430692230336 submission.py:307] 88) loss = 0.132, grad_norm = 0.500 +I0914 23:05:02.521898 140387389703936 logging_writer.py:48] [89] global_step=89, grad_norm=0.0280385, loss=0.130329 +I0914 23:05:02.524749 140430692230336 submission.py:307] 89) loss = 0.130, grad_norm = 0.028 +I0914 23:05:04.580273 140387398096640 logging_writer.py:48] [90] global_step=90, grad_norm=0.499999, loss=0.132278 +I0914 23:05:04.583293 140430692230336 submission.py:307] 90) loss = 0.132, grad_norm = 0.500 +I0914 23:05:06.685330 140387389703936 logging_writer.py:48] [91] global_step=91, grad_norm=0.443706, loss=0.130978 +I0914 23:05:06.688234 140430692230336 submission.py:307] 91) loss = 0.131, grad_norm = 0.444 +I0914 23:05:08.925490 140387398096640 logging_writer.py:48] [92] global_step=92, grad_norm=0.174317, loss=0.128247 +I0914 23:05:08.928274 140430692230336 submission.py:307] 92) loss = 0.128, grad_norm = 0.174 +I0914 23:05:10.833920 140387389703936 logging_writer.py:48] [93] global_step=93, grad_norm=0.0411293, loss=0.128909 +I0914 23:05:10.836707 140430692230336 submission.py:307] 93) loss = 0.129, grad_norm = 0.041 +I0914 23:05:12.942585 140387398096640 logging_writer.py:48] [94] global_step=94, grad_norm=0.229357, loss=0.130899 +I0914 23:05:12.945359 140430692230336 submission.py:307] 94) loss = 0.131, grad_norm = 0.229 +I0914 23:05:15.547581 140387389703936 logging_writer.py:48] [95] global_step=95, grad_norm=0.499999, loss=0.128129 +I0914 23:05:15.550989 140430692230336 submission.py:307] 95) loss = 0.128, grad_norm = 0.500 +I0914 23:05:17.705840 140387398096640 logging_writer.py:48] [96] global_step=96, grad_norm=0.499999, loss=0.130716 +I0914 23:05:17.708619 140430692230336 submission.py:307] 96) loss = 0.131, grad_norm = 0.500 +I0914 23:05:20.102333 140387389703936 logging_writer.py:48] [97] global_step=97, grad_norm=0.0999199, loss=0.128044 +I0914 23:05:20.105106 140430692230336 submission.py:307] 97) loss = 0.128, grad_norm = 0.100 +I0914 23:05:22.045558 140387398096640 logging_writer.py:48] [98] global_step=98, grad_norm=0.5, loss=0.132737 +I0914 23:05:22.048346 140430692230336 submission.py:307] 98) loss = 0.133, grad_norm = 0.500 +I0914 23:05:24.327086 140387389703936 logging_writer.py:48] [99] global_step=99, grad_norm=0.140322, loss=0.129019 +I0914 23:05:24.329942 140430692230336 submission.py:307] 99) loss = 0.129, grad_norm = 0.140 +I0914 23:05:26.544862 140387398096640 logging_writer.py:48] [100] global_step=100, grad_norm=0.385797, loss=0.127836 +I0914 23:05:26.547734 140430692230336 submission.py:307] 100) loss = 0.128, grad_norm = 0.386 +I0914 23:08:41.644727 140430692230336 spec.py:333] Evaluating on the training split. +I0914 23:19:17.896546 140430692230336 spec.py:346] Evaluating on the validation split. +I0914 23:22:52.292607 140430692230336 spec.py:363] Evaluating on the test split. +I0914 23:27:01.020378 140430692230336 submission_runner.py:516] Time since start: 3541.54s, Step: 191, {'train/loss': 0.12909747092187046, 'validation/loss': 0.12998375527518646, 'validation/num_examples': 83274637, 'test/loss': 0.1327623055301064, 'test/num_examples': 95000000, 'score': 368.1394748687744, 'total_duration': 3541.5391387939453, 'accumulated_submission_time': 368.1394748687744, 'accumulated_eval_time': 3172.300148487091, 'accumulated_logging_time': 0.09424114227294922} +I0914 23:27:01.158732 140387389703936 logging_writer.py:48] [191] accumulated_eval_time=3172.3, accumulated_logging_time=0.0942411, accumulated_submission_time=368.139, global_step=191, preemption_count=0, score=368.139, test/loss=0.132762, test/num_examples=95000000, total_duration=3541.54, train/loss=0.129097, validation/loss=0.129984, validation/num_examples=83274637 +I0914 23:32:58.463673 140430692230336 spec.py:333] Evaluating on the training split. +I0914 23:43:19.362220 140430692230336 spec.py:346] Evaluating on the validation split. +I0914 23:46:53.964073 140430692230336 spec.py:363] Evaluating on the test split. +I0914 23:50:58.595853 140430692230336 submission_runner.py:516] Time since start: 4979.11s, Step: 381, {'train/loss': 0.1258799900329567, 'validation/loss': 0.12747986184228655, 'validation/num_examples': 83274637, 'test/loss': 0.12999943839930483, 'test/num_examples': 95000000, 'score': 724.754016160965, 'total_duration': 4979.114598989487, 'accumulated_submission_time': 724.754016160965, 'accumulated_eval_time': 4252.43236041069, 'accumulated_logging_time': 0.239152193069458} +I0914 23:50:58.618936 140387398096640 logging_writer.py:48] [381] accumulated_eval_time=4252.43, accumulated_logging_time=0.239152, accumulated_submission_time=724.754, global_step=381, preemption_count=0, score=724.754, test/loss=0.129999, test/num_examples=95000000, total_duration=4979.11, train/loss=0.12588, validation/loss=0.12748, validation/num_examples=83274637 +I0914 23:53:57.327730 140387389703936 logging_writer.py:48] [500] global_step=500, grad_norm=0.0853532, loss=0.132422 +I0914 23:53:57.330902 140430692230336 submission.py:307] 500) loss = 0.132, grad_norm = 0.085 +I0914 23:56:55.252526 140430692230336 spec.py:333] Evaluating on the training split. +I0915 00:06:51.641778 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 00:10:25.516654 140430692230336 spec.py:363] Evaluating on the test split. +I0915 00:14:28.058506 140430692230336 submission_runner.py:516] Time since start: 6388.58s, Step: 582, {'train/loss': 0.12766485183422616, 'validation/loss': 0.1268212165814573, 'validation/num_examples': 83274637, 'test/loss': 0.1294382748072574, 'test/num_examples': 95000000, 'score': 1080.6785175800323, 'total_duration': 6388.577245473862, 'accumulated_submission_time': 1080.6785175800323, 'accumulated_eval_time': 5305.238393068314, 'accumulated_logging_time': 0.26903295516967773} +I0915 00:14:28.080744 140387398096640 logging_writer.py:48] [582] accumulated_eval_time=5305.24, accumulated_logging_time=0.269033, accumulated_submission_time=1080.68, global_step=582, preemption_count=0, score=1080.68, test/loss=0.129438, test/num_examples=95000000, total_duration=6388.58, train/loss=0.127665, validation/loss=0.126821, validation/num_examples=83274637 +I0915 00:20:24.693846 140430692230336 spec.py:333] Evaluating on the training split. +I0915 00:29:27.626880 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 00:33:01.343981 140430692230336 spec.py:363] Evaluating on the test split. +I0915 00:37:04.043601 140430692230336 submission_runner.py:516] Time since start: 7744.56s, Step: 799, {'train/loss': 0.12682287451323213, 'validation/loss': 0.1262535467938525, 'validation/num_examples': 83274637, 'test/loss': 0.1286774359707481, 'test/num_examples': 95000000, 'score': 1436.580223083496, 'total_duration': 7744.562363862991, 'accumulated_submission_time': 1436.580223083496, 'accumulated_eval_time': 6304.588199138641, 'accumulated_logging_time': 0.29724669456481934} +I0915 00:37:04.065424 140387389703936 logging_writer.py:48] [799] accumulated_eval_time=6304.59, accumulated_logging_time=0.297247, accumulated_submission_time=1436.58, global_step=799, preemption_count=0, score=1436.58, test/loss=0.128677, test/num_examples=95000000, total_duration=7744.56, train/loss=0.126823, validation/loss=0.126254, validation/num_examples=83274637 +I0915 00:42:52.119241 140387398096640 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.0104396, loss=0.121036 +I0915 00:42:52.122713 140430692230336 submission.py:307] 1000) loss = 0.121, grad_norm = 0.010 +I0915 00:43:00.398355 140430692230336 spec.py:333] Evaluating on the training split. +I0915 00:52:46.453541 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 00:56:19.699054 140430692230336 spec.py:363] Evaluating on the test split. +I0915 01:00:21.585541 140430692230336 submission_runner.py:516] Time since start: 9142.10s, Step: 1005, {'train/loss': 0.12701274664200785, 'validation/loss': 0.12602205896306515, 'validation/num_examples': 83274637, 'test/loss': 0.12834169397502698, 'test/num_examples': 95000000, 'score': 1792.2344620227814, 'total_duration': 9142.104288339615, 'accumulated_submission_time': 1792.2344620227814, 'accumulated_eval_time': 7345.77540397644, 'accumulated_logging_time': 0.32527947425842285} +I0915 01:00:21.608212 140387389703936 logging_writer.py:48] [1005] accumulated_eval_time=7345.78, accumulated_logging_time=0.325279, accumulated_submission_time=1792.23, global_step=1005, preemption_count=0, score=1792.23, test/loss=0.128342, test/num_examples=95000000, total_duration=9142.1, train/loss=0.127013, validation/loss=0.126022, validation/num_examples=83274637 +I0915 01:06:18.129926 140430692230336 spec.py:333] Evaluating on the training split. +I0915 01:15:18.855147 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 01:18:51.701303 140430692230336 spec.py:363] Evaluating on the test split. +I0915 01:22:52.105368 140430692230336 submission_runner.py:516] Time since start: 10492.62s, Step: 1211, {'train/loss': 0.1238404780887573, 'validation/loss': 0.12567188756322203, 'validation/num_examples': 83274637, 'test/loss': 0.12807911515599302, 'test/num_examples': 95000000, 'score': 2148.102383852005, 'total_duration': 10492.624114513397, 'accumulated_submission_time': 2148.102383852005, 'accumulated_eval_time': 8339.750897169113, 'accumulated_logging_time': 0.3561697006225586} +I0915 01:22:52.126776 140387398096640 logging_writer.py:48] [1211] accumulated_eval_time=8339.75, accumulated_logging_time=0.35617, accumulated_submission_time=2148.1, global_step=1211, preemption_count=0, score=2148.1, test/loss=0.128079, test/num_examples=95000000, total_duration=10492.6, train/loss=0.12384, validation/loss=0.125672, validation/num_examples=83274637 +I0915 01:28:49.266469 140430692230336 spec.py:333] Evaluating on the training split. +I0915 01:38:03.022549 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 01:41:34.572609 140430692230336 spec.py:363] Evaluating on the test split. +I0915 01:45:32.003397 140430692230336 submission_runner.py:516] Time since start: 11852.52s, Step: 1425, {'train/loss': 0.12282566049235345, 'validation/loss': 0.1254992503295916, 'validation/num_examples': 83274637, 'test/loss': 0.12776067463925012, 'test/num_examples': 95000000, 'score': 2504.586869239807, 'total_duration': 11852.522161483765, 'accumulated_submission_time': 2504.586869239807, 'accumulated_eval_time': 9342.487866640091, 'accumulated_logging_time': 0.38373899459838867} +I0915 01:45:32.025853 140387389703936 logging_writer.py:48] [1425] accumulated_eval_time=9342.49, accumulated_logging_time=0.383739, accumulated_submission_time=2504.59, global_step=1425, preemption_count=0, score=2504.59, test/loss=0.127761, test/num_examples=95000000, total_duration=11852.5, train/loss=0.122826, validation/loss=0.125499, validation/num_examples=83274637 +I0915 01:47:05.837143 140387398096640 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.0444963, loss=0.123239 +I0915 01:47:05.840025 140430692230336 submission.py:307] 1500) loss = 0.123, grad_norm = 0.044 +I0915 01:51:28.865372 140430692230336 spec.py:333] Evaluating on the training split. +I0915 01:59:06.250723 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 02:02:35.272696 140430692230336 spec.py:363] Evaluating on the test split. +I0915 02:06:27.814889 140430692230336 submission_runner.py:516] Time since start: 13108.33s, Step: 1640, {'train/loss': 0.12378746897426325, 'validation/loss': 0.12538934788679798, 'validation/num_examples': 83274637, 'test/loss': 0.12786424584687886, 'test/num_examples': 95000000, 'score': 2860.7693309783936, 'total_duration': 13108.333644151688, 'accumulated_submission_time': 2860.7693309783936, 'accumulated_eval_time': 10241.437473535538, 'accumulated_logging_time': 0.4124159812927246} +I0915 02:06:27.836892 140387389703936 logging_writer.py:48] [1640] accumulated_eval_time=10241.4, accumulated_logging_time=0.412416, accumulated_submission_time=2860.77, global_step=1640, preemption_count=0, score=2860.77, test/loss=0.127864, test/num_examples=95000000, total_duration=13108.3, train/loss=0.123787, validation/loss=0.125389, validation/num_examples=83274637 +I0915 02:12:25.116022 140430692230336 spec.py:333] Evaluating on the training split. +I0915 02:21:14.654291 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 02:24:39.033711 140430692230336 spec.py:363] Evaluating on the test split. +I0915 02:28:21.718361 140430692230336 submission_runner.py:516] Time since start: 14422.24s, Step: 1865, {'train/loss': 0.12348188611105475, 'validation/loss': 0.12534701745935206, 'validation/num_examples': 83274637, 'test/loss': 0.12775242581401625, 'test/num_examples': 95000000, 'score': 3217.3857910633087, 'total_duration': 14422.237087965012, 'accumulated_submission_time': 3217.3857910633087, 'accumulated_eval_time': 11198.039835691452, 'accumulated_logging_time': 0.4407162666320801} +I0915 02:28:21.741493 140387398096640 logging_writer.py:48] [1865] accumulated_eval_time=11198, accumulated_logging_time=0.440716, accumulated_submission_time=3217.39, global_step=1865, preemption_count=0, score=3217.39, test/loss=0.127752, test/num_examples=95000000, total_duration=14422.2, train/loss=0.123482, validation/loss=0.125347, validation/num_examples=83274637 +I0915 02:31:05.847508 140387389703936 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.0298743, loss=0.121125 +I0915 02:31:05.850475 140430692230336 submission.py:307] 2000) loss = 0.121, grad_norm = 0.030 +I0915 02:34:19.410855 140430692230336 spec.py:333] Evaluating on the training split. +I0915 02:41:43.787057 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 02:44:58.443597 140430692230336 spec.py:363] Evaluating on the test split. +I0915 02:48:20.103129 140430692230336 submission_runner.py:516] Time since start: 15620.62s, Step: 2110, {'train/loss': 0.12460651552788701, 'validation/loss': 0.12529222731759918, 'validation/num_examples': 83274637, 'test/loss': 0.12772872356069465, 'test/num_examples': 95000000, 'score': 3574.3835921287537, 'total_duration': 15620.62188577652, 'accumulated_submission_time': 3574.3835921287537, 'accumulated_eval_time': 12038.732187271118, 'accumulated_logging_time': 0.4701969623565674} +I0915 02:48:20.126259 140387398096640 logging_writer.py:48] [2110] accumulated_eval_time=12038.7, accumulated_logging_time=0.470197, accumulated_submission_time=3574.38, global_step=2110, preemption_count=0, score=3574.38, test/loss=0.127729, test/num_examples=95000000, total_duration=15620.6, train/loss=0.124607, validation/loss=0.125292, validation/num_examples=83274637 +I0915 02:54:17.422877 140430692230336 spec.py:333] Evaluating on the training split. +I0915 03:00:44.089213 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 03:03:38.713424 140430692230336 spec.py:363] Evaluating on the test split. +I0915 03:06:18.743510 140430692230336 submission_runner.py:516] Time since start: 16699.26s, Step: 2343, {'train/loss': 0.1233789449432704, 'validation/loss': 0.12505935572070498, 'validation/num_examples': 83274637, 'test/loss': 0.12741919558362458, 'test/num_examples': 95000000, 'score': 3931.008520126343, 'total_duration': 16699.2622692585, 'accumulated_submission_time': 3931.008520126343, 'accumulated_eval_time': 12760.052886724472, 'accumulated_logging_time': 0.49965381622314453} +I0915 03:06:18.772725 140387389703936 logging_writer.py:48] [2343] accumulated_eval_time=12760.1, accumulated_logging_time=0.499654, accumulated_submission_time=3931.01, global_step=2343, preemption_count=0, score=3931.01, test/loss=0.127419, test/num_examples=95000000, total_duration=16699.3, train/loss=0.123379, validation/loss=0.125059, validation/num_examples=83274637 +I0915 03:09:54.862778 140387398096640 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.0113501, loss=0.124823 +I0915 03:09:54.865822 140430692230336 submission.py:307] 2500) loss = 0.125, grad_norm = 0.011 +I0915 03:12:15.163441 140430692230336 spec.py:333] Evaluating on the training split. +I0915 03:16:38.403301 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 03:18:48.342651 140430692230336 spec.py:363] Evaluating on the test split. +I0915 03:20:49.387460 140430692230336 submission_runner.py:516] Time since start: 17569.91s, Step: 2572, {'train/loss': 0.1220150500905709, 'validation/loss': 0.12465657975991778, 'validation/num_examples': 83274637, 'test/loss': 0.12701075822079308, 'test/num_examples': 95000000, 'score': 4286.738893270493, 'total_duration': 17569.906219482422, 'accumulated_submission_time': 4286.738893270493, 'accumulated_eval_time': 13274.276980161667, 'accumulated_logging_time': 0.5351388454437256} +I0915 03:20:49.410027 140387389703936 logging_writer.py:48] [2572] accumulated_eval_time=13274.3, accumulated_logging_time=0.535139, accumulated_submission_time=4286.74, global_step=2572, preemption_count=0, score=4286.74, test/loss=0.127011, test/num_examples=95000000, total_duration=17569.9, train/loss=0.122015, validation/loss=0.124657, validation/num_examples=83274637 +I0915 03:26:47.014801 140430692230336 spec.py:333] Evaluating on the training split. +I0915 03:29:33.829569 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 03:31:00.345729 140430692230336 spec.py:363] Evaluating on the test split. +I0915 03:32:16.433936 140430692230336 submission_runner.py:516] Time since start: 18256.95s, Step: 2802, {'train/loss': 0.12319279921402525, 'validation/loss': 0.12489396141441854, 'validation/num_examples': 83274637, 'test/loss': 0.127454749205258, 'test/num_examples': 95000000, 'score': 4643.671773433685, 'total_duration': 18256.95265340805, 'accumulated_submission_time': 4643.671773433685, 'accumulated_eval_time': 13603.696100711823, 'accumulated_logging_time': 0.5641770362854004} +I0915 03:32:16.456236 140387398096640 logging_writer.py:48] [2802] accumulated_eval_time=13603.7, accumulated_logging_time=0.564177, accumulated_submission_time=4643.67, global_step=2802, preemption_count=0, score=4643.67, test/loss=0.127455, test/num_examples=95000000, total_duration=18257, train/loss=0.123193, validation/loss=0.124894, validation/num_examples=83274637 +I0915 03:37:36.933861 140387389703936 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.0185175, loss=0.123084 +I0915 03:37:36.936878 140430692230336 submission.py:307] 3000) loss = 0.123, grad_norm = 0.019 +I0915 03:38:14.363247 140430692230336 spec.py:333] Evaluating on the training split. +I0915 03:39:45.050985 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 03:40:43.462168 140430692230336 spec.py:363] Evaluating on the test split. +I0915 03:41:50.312085 140430692230336 submission_runner.py:516] Time since start: 18830.83s, Step: 3020, {'train/loss': 0.12243054694558941, 'validation/loss': 0.12509942581828104, 'validation/num_examples': 83274637, 'test/loss': 0.12761369219633403, 'test/num_examples': 95000000, 'score': 5000.901176691055, 'total_duration': 18830.83085513115, 'accumulated_submission_time': 5000.901176691055, 'accumulated_eval_time': 13819.645005702972, 'accumulated_logging_time': 0.5928189754486084} +I0915 03:41:50.334920 140387398096640 logging_writer.py:48] [3020] accumulated_eval_time=13819.6, accumulated_logging_time=0.592819, accumulated_submission_time=5000.9, global_step=3020, preemption_count=0, score=5000.9, test/loss=0.127614, test/num_examples=95000000, total_duration=18830.8, train/loss=0.122431, validation/loss=0.125099, validation/num_examples=83274637 +I0915 03:47:48.055094 140430692230336 spec.py:333] Evaluating on the training split. +I0915 03:48:55.947328 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 03:49:54.040784 140430692230336 spec.py:363] Evaluating on the test split. +I0915 03:51:00.749458 140430692230336 submission_runner.py:516] Time since start: 19381.27s, Step: 3262, {'train/loss': 0.12204526610523697, 'validation/loss': 0.12501785966782103, 'validation/num_examples': 83274637, 'test/loss': 0.1275428946753251, 'test/num_examples': 95000000, 'score': 5357.929572105408, 'total_duration': 19381.26821088791, 'accumulated_submission_time': 5357.929572105408, 'accumulated_eval_time': 14012.339433670044, 'accumulated_logging_time': 0.6221432685852051} +I0915 03:51:00.772340 140387389703936 logging_writer.py:48] [3262] accumulated_eval_time=14012.3, accumulated_logging_time=0.622143, accumulated_submission_time=5357.93, global_step=3262, preemption_count=0, score=5357.93, test/loss=0.127543, test/num_examples=95000000, total_duration=19381.3, train/loss=0.122045, validation/loss=0.125018, validation/num_examples=83274637 +I0915 03:56:58.035090 140430692230336 spec.py:333] Evaluating on the training split. +I0915 03:57:59.052765 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 03:58:57.337307 140430692230336 spec.py:363] Evaluating on the test split. +I0915 04:00:04.298829 140430692230336 submission_runner.py:516] Time since start: 19924.82s, Step: 3500, {'train/loss': 0.1239425728552684, 'validation/loss': 0.12483808297920312, 'validation/num_examples': 83274637, 'test/loss': 0.12739757180320338, 'test/num_examples': 95000000, 'score': 5714.501089096069, 'total_duration': 19924.81757092476, 'accumulated_submission_time': 5714.501089096069, 'accumulated_eval_time': 14198.603212356567, 'accumulated_logging_time': 0.6516408920288086} +I0915 04:00:04.321601 140387398096640 logging_writer.py:48] [3500] accumulated_eval_time=14198.6, accumulated_logging_time=0.651641, accumulated_submission_time=5714.5, global_step=3500, preemption_count=0, score=5714.5, test/loss=0.127398, test/num_examples=95000000, total_duration=19924.8, train/loss=0.123943, validation/loss=0.124838, validation/num_examples=83274637 +I0915 04:00:04.873746 140387389703936 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.035451, loss=0.121755 +I0915 04:00:04.876606 140430692230336 submission.py:307] 3500) loss = 0.122, grad_norm = 0.035 +I0915 04:06:00.598782 140430692230336 spec.py:333] Evaluating on the training split. +I0915 04:06:58.784576 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 04:07:57.077200 140430692230336 spec.py:363] Evaluating on the test split. +I0915 04:09:03.960178 140430692230336 submission_runner.py:516] Time since start: 20464.48s, Step: 3717, {'train/loss': 0.12529232250953456, 'validation/loss': 0.12474395436402373, 'validation/num_examples': 83274637, 'test/loss': 0.12714521083486457, 'test/num_examples': 95000000, 'score': 6070.098887205124, 'total_duration': 20464.478921175003, 'accumulated_submission_time': 6070.098887205124, 'accumulated_eval_time': 14381.964620828629, 'accumulated_logging_time': 0.6807894706726074} +I0915 04:09:03.982598 140387398096640 logging_writer.py:48] [3717] accumulated_eval_time=14382, accumulated_logging_time=0.680789, accumulated_submission_time=6070.1, global_step=3717, preemption_count=0, score=6070.1, test/loss=0.127145, test/num_examples=95000000, total_duration=20464.5, train/loss=0.125292, validation/loss=0.124744, validation/num_examples=83274637 +I0915 04:15:00.552976 140430692230336 spec.py:333] Evaluating on the training split. +I0915 04:15:58.781959 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 04:16:56.974576 140430692230336 spec.py:363] Evaluating on the test split. +I0915 04:18:03.851663 140430692230336 submission_runner.py:516] Time since start: 21004.37s, Step: 3926, {'train/loss': 0.12420619381989272, 'validation/loss': 0.12467618500307047, 'validation/num_examples': 83274637, 'test/loss': 0.12713146270671644, 'test/num_examples': 95000000, 'score': 6425.994694471359, 'total_duration': 21004.370398521423, 'accumulated_submission_time': 6425.994694471359, 'accumulated_eval_time': 14565.263352394104, 'accumulated_logging_time': 0.7095613479614258} +I0915 04:18:03.873322 140387389703936 logging_writer.py:48] [3926] accumulated_eval_time=14565.3, accumulated_logging_time=0.709561, accumulated_submission_time=6425.99, global_step=3926, preemption_count=0, score=6425.99, test/loss=0.127131, test/num_examples=95000000, total_duration=21004.4, train/loss=0.124206, validation/loss=0.124676, validation/num_examples=83274637 +I0915 04:19:17.515185 140387398096640 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.0418413, loss=0.121179 +I0915 04:19:17.518374 140430692230336 submission.py:307] 4000) loss = 0.121, grad_norm = 0.042 +I0915 04:24:00.871923 140430692230336 spec.py:333] Evaluating on the training split. +I0915 04:24:58.923568 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 04:25:57.170121 140430692230336 spec.py:363] Evaluating on the test split. +I0915 04:27:03.922461 140430692230336 submission_runner.py:516] Time since start: 21544.44s, Step: 4167, {'train/loss': 0.12163764439701313, 'validation/loss': 0.12446432338516199, 'validation/num_examples': 83274637, 'test/loss': 0.12677098810601486, 'test/num_examples': 95000000, 'score': 6782.304340839386, 'total_duration': 21544.441210746765, 'accumulated_submission_time': 6782.304340839386, 'accumulated_eval_time': 14748.313912153244, 'accumulated_logging_time': 0.7375872135162354} +I0915 04:27:03.945153 140387389703936 logging_writer.py:48] [4167] accumulated_eval_time=14748.3, accumulated_logging_time=0.737587, accumulated_submission_time=6782.3, global_step=4167, preemption_count=0, score=6782.3, test/loss=0.126771, test/num_examples=95000000, total_duration=21544.4, train/loss=0.121638, validation/loss=0.124464, validation/num_examples=83274637 +I0915 04:33:01.923356 140430692230336 spec.py:333] Evaluating on the training split. +I0915 04:33:59.827790 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 04:34:57.937195 140430692230336 spec.py:363] Evaluating on the test split. +I0915 04:36:04.564912 140430692230336 submission_runner.py:516] Time since start: 22085.08s, Step: 4385, {'train/loss': 0.1238337018820797, 'validation/loss': 0.12463919110672007, 'validation/num_examples': 83274637, 'test/loss': 0.1270552308603387, 'test/num_examples': 95000000, 'score': 7139.600712776184, 'total_duration': 22085.083664894104, 'accumulated_submission_time': 7139.600712776184, 'accumulated_eval_time': 14930.95549082756, 'accumulated_logging_time': 0.7668135166168213} +I0915 04:36:04.589308 140387398096640 logging_writer.py:48] [4385] accumulated_eval_time=14931, accumulated_logging_time=0.766814, accumulated_submission_time=7139.6, global_step=4385, preemption_count=0, score=7139.6, test/loss=0.127055, test/num_examples=95000000, total_duration=22085.1, train/loss=0.123834, validation/loss=0.124639, validation/num_examples=83274637 +I0915 04:39:03.537482 140387389703936 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.0251874, loss=0.116085 +I0915 04:39:03.540636 140430692230336 submission.py:307] 4500) loss = 0.116, grad_norm = 0.025 +I0915 04:42:02.363955 140430692230336 spec.py:333] Evaluating on the training split. +I0915 04:43:00.585655 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 04:43:58.962604 140430692230336 spec.py:363] Evaluating on the test split. +I0915 04:45:05.766148 140430692230336 submission_runner.py:516] Time since start: 22626.28s, Step: 4603, {'train/loss': 0.12345669351000732, 'validation/loss': 0.12454528752425037, 'validation/num_examples': 83274637, 'test/loss': 0.12706868935562937, 'test/num_examples': 95000000, 'score': 7496.691490888596, 'total_duration': 22626.284920215607, 'accumulated_submission_time': 7496.691490888596, 'accumulated_eval_time': 15114.357768058777, 'accumulated_logging_time': 0.7977817058563232} +I0915 04:45:05.790121 140387398096640 logging_writer.py:48] [4603] accumulated_eval_time=15114.4, accumulated_logging_time=0.797782, accumulated_submission_time=7496.69, global_step=4603, preemption_count=0, score=7496.69, test/loss=0.127069, test/num_examples=95000000, total_duration=22626.3, train/loss=0.123457, validation/loss=0.124545, validation/num_examples=83274637 +I0915 04:51:03.321946 140430692230336 spec.py:333] Evaluating on the training split. +I0915 04:52:01.217181 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 04:52:59.224866 140430692230336 spec.py:363] Evaluating on the test split. +I0915 04:54:06.087891 140430692230336 submission_runner.py:516] Time since start: 23166.61s, Step: 4804, {'train/loss': 0.12165446601974438, 'validation/loss': 0.12426333800135851, 'validation/num_examples': 83274637, 'test/loss': 0.12664656422054893, 'test/num_examples': 95000000, 'score': 7853.557901144028, 'total_duration': 23166.60662317276, 'accumulated_submission_time': 7853.557901144028, 'accumulated_eval_time': 15297.123735904694, 'accumulated_logging_time': 0.8283920288085938} +I0915 04:54:06.110568 140387389703936 logging_writer.py:48] [4804] accumulated_eval_time=15297.1, accumulated_logging_time=0.828392, accumulated_submission_time=7853.56, global_step=4804, preemption_count=0, score=7853.56, test/loss=0.126647, test/num_examples=95000000, total_duration=23166.6, train/loss=0.121654, validation/loss=0.124263, validation/num_examples=83274637 +I0915 04:59:02.474852 140387398096640 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.0292757, loss=0.121053 +I0915 04:59:02.479732 140430692230336 submission.py:307] 5000) loss = 0.121, grad_norm = 0.029 +I0915 05:00:03.625482 140430692230336 spec.py:333] Evaluating on the training split. +I0915 05:01:01.911742 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 05:02:00.370590 140430692230336 spec.py:363] Evaluating on the test split. +I0915 05:03:07.390422 140430692230336 submission_runner.py:516] Time since start: 23707.91s, Step: 5036, {'train/loss': 0.12223824308739915, 'validation/loss': 0.12430292438670795, 'validation/num_examples': 83274637, 'test/loss': 0.12667685665363512, 'test/num_examples': 95000000, 'score': 8210.377913236618, 'total_duration': 23707.909193515778, 'accumulated_submission_time': 8210.377913236618, 'accumulated_eval_time': 15480.888777971268, 'accumulated_logging_time': 0.857440710067749} +I0915 05:03:07.413786 140387389703936 logging_writer.py:48] [5036] accumulated_eval_time=15480.9, accumulated_logging_time=0.857441, accumulated_submission_time=8210.38, global_step=5036, preemption_count=0, score=8210.38, test/loss=0.126677, test/num_examples=95000000, total_duration=23707.9, train/loss=0.122238, validation/loss=0.124303, validation/num_examples=83274637 +I0915 05:09:03.765746 140430692230336 spec.py:333] Evaluating on the training split. +I0915 05:10:01.665147 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 05:10:59.959233 140430692230336 spec.py:363] Evaluating on the test split. +I0915 05:12:06.749243 140430692230336 submission_runner.py:516] Time since start: 24247.27s, Step: 5244, {'train/loss': 0.12157833373759286, 'validation/loss': 0.12443081815811288, 'validation/num_examples': 83274637, 'test/loss': 0.12688449270710192, 'test/num_examples': 95000000, 'score': 8566.05491733551, 'total_duration': 24247.26802277565, 'accumulated_submission_time': 8566.05491733551, 'accumulated_eval_time': 15663.872374534607, 'accumulated_logging_time': 0.8873980045318604} +I0915 05:12:06.772192 140387398096640 logging_writer.py:48] [5244] accumulated_eval_time=15663.9, accumulated_logging_time=0.887398, accumulated_submission_time=8566.05, global_step=5244, preemption_count=0, score=8566.05, test/loss=0.126884, test/num_examples=95000000, total_duration=24247.3, train/loss=0.121578, validation/loss=0.124431, validation/num_examples=83274637 +I0915 05:18:03.114362 140430692230336 spec.py:333] Evaluating on the training split. +I0915 05:19:01.008723 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 05:19:58.938573 140430692230336 spec.py:363] Evaluating on the test split. +I0915 05:21:05.633595 140430692230336 submission_runner.py:516] Time since start: 24786.15s, Step: 5455, {'train/loss': 0.12259185467902575, 'validation/loss': 0.12423333122155497, 'validation/num_examples': 83274637, 'test/loss': 0.1265792360459177, 'test/num_examples': 95000000, 'score': 8921.71652340889, 'total_duration': 24786.152362585068, 'accumulated_submission_time': 8921.71652340889, 'accumulated_eval_time': 15846.391675710678, 'accumulated_logging_time': 0.9166655540466309} +I0915 05:21:05.658561 140387389703936 logging_writer.py:48] [5455] accumulated_eval_time=15846.4, accumulated_logging_time=0.916666, accumulated_submission_time=8921.72, global_step=5455, preemption_count=0, score=8921.72, test/loss=0.126579, test/num_examples=95000000, total_duration=24786.2, train/loss=0.122592, validation/loss=0.124233, validation/num_examples=83274637 +I0915 05:21:47.342798 140387398096640 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.0171384, loss=0.120783 +I0915 05:21:47.345771 140430692230336 submission.py:307] 5500) loss = 0.121, grad_norm = 0.017 +I0915 05:27:03.559641 140430692230336 spec.py:333] Evaluating on the training split. +I0915 05:28:01.781203 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 05:29:00.104547 140430692230336 spec.py:363] Evaluating on the test split. +I0915 05:30:06.954770 140430692230336 submission_runner.py:516] Time since start: 25327.47s, Step: 5681, {'train/loss': 0.12107022227922803, 'validation/loss': 0.12432816125522116, 'validation/num_examples': 83274637, 'test/loss': 0.12672340302589818, 'test/num_examples': 95000000, 'score': 9278.934896945953, 'total_duration': 25327.473544597626, 'accumulated_submission_time': 9278.934896945953, 'accumulated_eval_time': 16029.786890268326, 'accumulated_logging_time': 0.9480431079864502} +I0915 05:30:06.978257 140387389703936 logging_writer.py:48] [5681] accumulated_eval_time=16029.8, accumulated_logging_time=0.948043, accumulated_submission_time=9278.93, global_step=5681, preemption_count=0, score=9278.93, test/loss=0.126723, test/num_examples=95000000, total_duration=25327.5, train/loss=0.12107, validation/loss=0.124328, validation/num_examples=83274637 +I0915 05:36:04.035876 140430692230336 spec.py:333] Evaluating on the training split. +I0915 05:37:02.438550 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 05:38:00.929816 140430692230336 spec.py:363] Evaluating on the test split. +I0915 05:39:07.668882 140430692230336 submission_runner.py:516] Time since start: 25868.19s, Step: 5893, {'train/loss': 0.12172740083477311, 'validation/loss': 0.12428990918299418, 'validation/num_examples': 83274637, 'test/loss': 0.12659455729675292, 'test/num_examples': 95000000, 'score': 9635.31711769104, 'total_duration': 25868.187648296356, 'accumulated_submission_time': 9635.31711769104, 'accumulated_eval_time': 16213.41997885704, 'accumulated_logging_time': 0.9778847694396973} +I0915 05:39:07.693088 140387398096640 logging_writer.py:48] [5893] accumulated_eval_time=16213.4, accumulated_logging_time=0.977885, accumulated_submission_time=9635.32, global_step=5893, preemption_count=0, score=9635.32, test/loss=0.126595, test/num_examples=95000000, total_duration=25868.2, train/loss=0.121727, validation/loss=0.12429, validation/num_examples=83274637 +I0915 05:41:59.206768 140387389703936 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.0231908, loss=0.120849 +I0915 05:41:59.209823 140430692230336 submission.py:307] 6000) loss = 0.121, grad_norm = 0.023 +I0915 05:45:04.563751 140430692230336 spec.py:333] Evaluating on the training split. +I0915 05:46:02.683717 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 05:47:00.989512 140430692230336 spec.py:363] Evaluating on the test split. +I0915 05:48:07.778337 140430692230336 submission_runner.py:516] Time since start: 26408.30s, Step: 6094, {'train/loss': 0.12039696787248182, 'validation/loss': 0.12430679226261986, 'validation/num_examples': 83274637, 'test/loss': 0.12671427263424523, 'test/num_examples': 95000000, 'score': 9991.516325712204, 'total_duration': 26408.29709792137, 'accumulated_submission_time': 9991.516325712204, 'accumulated_eval_time': 16396.634641885757, 'accumulated_logging_time': 1.0087382793426514} +I0915 05:48:07.801823 140387398096640 logging_writer.py:48] [6094] accumulated_eval_time=16396.6, accumulated_logging_time=1.00874, accumulated_submission_time=9991.52, global_step=6094, preemption_count=0, score=9991.52, test/loss=0.126714, test/num_examples=95000000, total_duration=26408.3, train/loss=0.120397, validation/loss=0.124307, validation/num_examples=83274637 +I0915 05:54:04.835966 140430692230336 spec.py:333] Evaluating on the training split. +I0915 05:55:02.820064 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 05:56:00.971880 140430692230336 spec.py:363] Evaluating on the test split. +I0915 05:57:07.680451 140430692230336 submission_runner.py:516] Time since start: 26948.20s, Step: 6306, {'train/loss': 0.12273077348317894, 'validation/loss': 0.12418806031891497, 'validation/num_examples': 83274637, 'test/loss': 0.12653463702199835, 'test/num_examples': 95000000, 'score': 10347.876336097717, 'total_duration': 26948.199214220047, 'accumulated_submission_time': 10347.876336097717, 'accumulated_eval_time': 16579.479200601578, 'accumulated_logging_time': 1.0386712551116943} +I0915 05:57:07.704881 140387389703936 logging_writer.py:48] [6306] accumulated_eval_time=16579.5, accumulated_logging_time=1.03867, accumulated_submission_time=10347.9, global_step=6306, preemption_count=0, score=10347.9, test/loss=0.126535, test/num_examples=95000000, total_duration=26948.2, train/loss=0.122731, validation/loss=0.124188, validation/num_examples=83274637 +I0915 06:02:29.442329 140387398096640 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.0182925, loss=0.115627 +I0915 06:02:29.445437 140430692230336 submission.py:307] 6500) loss = 0.116, grad_norm = 0.018 +I0915 06:03:05.291252 140430692230336 spec.py:333] Evaluating on the training split. +I0915 06:04:03.513678 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 06:05:01.892437 140430692230336 spec.py:363] Evaluating on the test split. +I0915 06:06:08.767637 140430692230336 submission_runner.py:516] Time since start: 27489.29s, Step: 6519, {'train/loss': 0.12139838587543274, 'validation/loss': 0.12410659305994814, 'validation/num_examples': 83274637, 'test/loss': 0.12643457385012977, 'test/num_examples': 95000000, 'score': 10704.78233575821, 'total_duration': 27489.286378383636, 'accumulated_submission_time': 10704.78233575821, 'accumulated_eval_time': 16762.955620765686, 'accumulated_logging_time': 1.0695700645446777} +I0915 06:06:08.791215 140387389703936 logging_writer.py:48] [6519] accumulated_eval_time=16763, accumulated_logging_time=1.06957, accumulated_submission_time=10704.8, global_step=6519, preemption_count=0, score=10704.8, test/loss=0.126435, test/num_examples=95000000, total_duration=27489.3, train/loss=0.121398, validation/loss=0.124107, validation/num_examples=83274637 +I0915 06:12:05.977186 140430692230336 spec.py:333] Evaluating on the training split. +I0915 06:13:04.502259 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 06:14:02.698162 140430692230336 spec.py:363] Evaluating on the test split. +I0915 06:15:09.541615 140430692230336 submission_runner.py:516] Time since start: 28030.06s, Step: 6766, {'train/loss': 0.12086757048271918, 'validation/loss': 0.12405763866279226, 'validation/num_examples': 83274637, 'test/loss': 0.12640602687843724, 'test/num_examples': 95000000, 'score': 11061.276944637299, 'total_duration': 28030.06036734581, 'accumulated_submission_time': 11061.276944637299, 'accumulated_eval_time': 16946.520087957382, 'accumulated_logging_time': 1.0994818210601807} +I0915 06:15:09.566431 140387398096640 logging_writer.py:48] [6766] accumulated_eval_time=16946.5, accumulated_logging_time=1.09948, accumulated_submission_time=11061.3, global_step=6766, preemption_count=0, score=11061.3, test/loss=0.126406, test/num_examples=95000000, total_duration=28030.1, train/loss=0.120868, validation/loss=0.124058, validation/num_examples=83274637 +I0915 06:21:07.582036 140430692230336 spec.py:333] Evaluating on the training split. +I0915 06:22:30.319368 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 06:23:30.602824 140430692230336 spec.py:363] Evaluating on the test split. +I0915 06:24:40.019474 140430692230336 submission_runner.py:516] Time since start: 28600.54s, Step: 6968, {'train/loss': 0.12346100642106007, 'validation/loss': 0.1240783339262097, 'validation/num_examples': 83274637, 'test/loss': 0.126419649889173, 'test/num_examples': 95000000, 'score': 11418.616167068481, 'total_duration': 28600.53822851181, 'accumulated_submission_time': 11418.616167068481, 'accumulated_eval_time': 17158.9575817585, 'accumulated_logging_time': 1.131810188293457} +I0915 06:24:40.043388 140387389703936 logging_writer.py:48] [6968] accumulated_eval_time=17159, accumulated_logging_time=1.13181, accumulated_submission_time=11418.6, global_step=6968, preemption_count=0, score=11418.6, test/loss=0.12642, test/num_examples=95000000, total_duration=28600.5, train/loss=0.123461, validation/loss=0.124078, validation/num_examples=83274637 +I0915 06:24:55.368246 140387398096640 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.00531687, loss=0.122083 +I0915 06:24:55.372456 140430692230336 submission.py:307] 7000) loss = 0.122, grad_norm = 0.005 +I0915 06:30:36.403476 140430692230336 spec.py:333] Evaluating on the training split. +I0915 06:32:14.043318 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 06:33:12.613742 140430692230336 spec.py:363] Evaluating on the test split. +I0915 06:34:19.467794 140430692230336 submission_runner.py:516] Time since start: 29179.99s, Step: 7182, {'train/loss': 0.12337468109549282, 'validation/loss': 0.12390680204242814, 'validation/num_examples': 83274637, 'test/loss': 0.1261435647168611, 'test/num_examples': 95000000, 'score': 11774.297981977463, 'total_duration': 29179.986558675766, 'accumulated_submission_time': 11774.297981977463, 'accumulated_eval_time': 17382.02194261551, 'accumulated_logging_time': 1.162308931350708} +I0915 06:34:19.490882 140387389703936 logging_writer.py:48] [7182] accumulated_eval_time=17382, accumulated_logging_time=1.16231, accumulated_submission_time=11774.3, global_step=7182, preemption_count=0, score=11774.3, test/loss=0.126144, test/num_examples=95000000, total_duration=29180, train/loss=0.123375, validation/loss=0.123907, validation/num_examples=83274637 +I0915 06:40:17.338822 140430692230336 spec.py:333] Evaluating on the training split. +I0915 06:41:27.681159 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 06:42:25.829924 140430692230336 spec.py:363] Evaluating on the test split. +I0915 06:43:32.434200 140430692230336 submission_runner.py:516] Time since start: 29732.95s, Step: 7383, {'train/loss': 0.12210321974814724, 'validation/loss': 0.12390433231433495, 'validation/num_examples': 83274637, 'test/loss': 0.12622765955521434, 'test/num_examples': 95000000, 'score': 12131.475723743439, 'total_duration': 29732.952960252762, 'accumulated_submission_time': 12131.475723743439, 'accumulated_eval_time': 17577.117379665375, 'accumulated_logging_time': 1.191917896270752} +I0915 06:43:32.459191 140387398096640 logging_writer.py:48] [7383] accumulated_eval_time=17577.1, accumulated_logging_time=1.19192, accumulated_submission_time=12131.5, global_step=7383, preemption_count=0, score=12131.5, test/loss=0.126228, test/num_examples=95000000, total_duration=29733, train/loss=0.122103, validation/loss=0.123904, validation/num_examples=83274637 +I0915 06:46:27.065687 140387389703936 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.00893666, loss=0.112597 +I0915 06:46:27.068716 140430692230336 submission.py:307] 7500) loss = 0.113, grad_norm = 0.009 +I0915 06:49:29.986634 140430692230336 spec.py:333] Evaluating on the training split. +I0915 06:50:37.170899 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 06:51:35.315681 140430692230336 spec.py:363] Evaluating on the test split. +I0915 06:52:42.043971 140430692230336 submission_runner.py:516] Time since start: 30282.56s, Step: 7592, {'train/loss': 0.12073468999321914, 'validation/loss': 0.1239396156478079, 'validation/num_examples': 83274637, 'test/loss': 0.12631729120788573, 'test/num_examples': 95000000, 'score': 12488.324034929276, 'total_duration': 30282.56272673607, 'accumulated_submission_time': 12488.324034929276, 'accumulated_eval_time': 17769.174802541733, 'accumulated_logging_time': 1.2236275672912598} +I0915 06:52:42.067519 140387398096640 logging_writer.py:48] [7592] accumulated_eval_time=17769.2, accumulated_logging_time=1.22363, accumulated_submission_time=12488.3, global_step=7592, preemption_count=0, score=12488.3, test/loss=0.126317, test/num_examples=95000000, total_duration=30282.6, train/loss=0.120735, validation/loss=0.12394, validation/num_examples=83274637 +I0915 06:58:39.013337 140430692230336 spec.py:333] Evaluating on the training split. +I0915 06:59:37.394168 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 07:00:35.815369 140430692230336 spec.py:363] Evaluating on the test split. +I0915 07:01:42.409938 140430692230336 submission_runner.py:516] Time since start: 30822.93s, Step: 7810, {'train/loss': 0.12293512181760188, 'validation/loss': 0.12395364025189097, 'validation/num_examples': 83274637, 'test/loss': 0.12631050040138647, 'test/num_examples': 95000000, 'score': 12844.588040351868, 'total_duration': 30822.928701400757, 'accumulated_submission_time': 12844.588040351868, 'accumulated_eval_time': 17952.57149863243, 'accumulated_logging_time': 1.2534687519073486} +I0915 07:01:42.433655 140387389703936 logging_writer.py:48] [7810] accumulated_eval_time=17952.6, accumulated_logging_time=1.25347, accumulated_submission_time=12844.6, global_step=7810, preemption_count=0, score=12844.6, test/loss=0.126311, test/num_examples=95000000, total_duration=30822.9, train/loss=0.122935, validation/loss=0.123954, validation/num_examples=83274637 +I0915 07:07:37.690515 140387398096640 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.00617268, loss=0.11821 +I0915 07:07:37.693492 140430692230336 submission.py:307] 8000) loss = 0.118, grad_norm = 0.006 +I0915 07:07:40.203955 140430692230336 spec.py:333] Evaluating on the training split. +I0915 07:08:49.109611 140430692230336 spec.py:346] Evaluating on the validation split. +I0915 07:09:47.266132 140430692230336 spec.py:363] Evaluating on the test split. +I0915 07:10:54.070553 140430692230336 submission_runner.py:516] Time since start: 31374.59s, Step: 8002, {'train/loss': 0.12136180953056898, 'validation/loss': 0.12385063270517058, 'validation/num_examples': 83274637, 'test/loss': 0.12613237252727308, 'test/num_examples': 95000000, 'score': 13201.688807487488, 'total_duration': 31374.58929824829, 'accumulated_submission_time': 13201.688807487488, 'accumulated_eval_time': 18146.438154935837, 'accumulated_logging_time': 1.283888578414917} +I0915 07:10:54.097156 140387389703936 logging_writer.py:48] [8002] accumulated_eval_time=18146.4, accumulated_logging_time=1.28389, accumulated_submission_time=13201.7, global_step=8002, preemption_count=0, score=13201.7, test/loss=0.126132, test/num_examples=95000000, total_duration=31374.6, train/loss=0.121362, validation/loss=0.123851, validation/num_examples=83274637 +I0915 07:16:51.034896 140387398096640 logging_writer.py:48] [8207] global_step=8207, preemption_count=0, score=13558.2 +I0915 07:16:56.349802 140430692230336 submission_runner.py:857] Final criteo1tb score: 13558.222728013992 diff --git a/logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..7c9c6de2 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,39 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/loss,test/num_examples,total_duration,train/loss,validation/loss,validation/num_examples +2072.9244389534,0.0,11.226190328598022,1,0,11.226190328598022,1.9351263324861223,95000000,2084.472838163376,1.9395356163303377,1.9414668747834607,83274637 +3172.300148487091,0.0942411422729492,368.1394748687744,191,0,368.1394748687744,0.1327623055301064,95000000,3541.5391387939453,0.1290974709218704,0.1299837552751864,83274637 +4252.43236041069,0.239152193069458,724.754016160965,381,0,724.754016160965,0.1299994383993048,95000000,4979.114598989487,0.1258799900329567,0.1274798618422865,83274637 +5305.238393068314,0.2690329551696777,1080.6785175800323,582,0,1080.6785175800323,0.1294382748072574,95000000,6388.577245473862,0.1276648518342261,0.1268212165814573,83274637 +6304.588199138641,0.2972466945648193,1436.580223083496,799,0,1436.580223083496,0.1286774359707481,95000000,7744.562363862991,0.1268228745132321,0.1262535467938525,83274637 +7345.77540397644,0.3252794742584228,1792.2344620227814,1005,0,1792.2344620227814,0.1283416939750269,95000000,9142.104288339617,0.1270127466420078,0.1260220589630651,83274637 +8339.750897169113,0.3561697006225586,2148.102383852005,1211,0,2148.102383852005,0.128079115155993,95000000,10492.624114513395,0.1238404780887573,0.125671887563222,83274637 +9342.487866640093,0.3837389945983886,2504.586869239807,1425,0,2504.586869239807,0.1277606746392501,95000000,11852.522161483765,0.1228256604923534,0.1254992503295916,83274637 +10241.437473535538,0.4124159812927246,2860.7693309783936,1640,0,2860.7693309783936,0.1278642458468788,95000000,13108.333644151688,0.1237874689742632,0.1253893478867979,83274637 +11198.039835691452,0.4407162666320801,3217.3857910633087,1865,0,3217.3857910633087,0.1277524258140162,95000000,14422.237087965012,0.1234818861110547,0.125347017459352,83274637 +12038.732187271118,0.4701969623565674,3574.3835921287537,2110,0,3574.3835921287537,0.1277287235606946,95000000,15620.62188577652,0.124606515527887,0.1252922273175991,83274637 +12760.052886724472,0.4996538162231445,3931.008520126343,2343,0,3931.008520126343,0.1274191955836245,95000000,16699.2622692585,0.1233789449432704,0.1250593557207049,83274637 +13274.276980161669,0.5351388454437256,4286.738893270493,2572,0,4286.738893270493,0.127010758220793,95000000,17569.906219482422,0.1220150500905709,0.1246565797599177,83274637 +13603.696100711824,0.5641770362854004,4643.671773433685,2802,0,4643.671773433685,0.127454749205258,95000000,18256.95265340805,0.1231927992140252,0.1248939614144185,83274637 +13819.645005702972,0.5928189754486084,5000.901176691055,3020,0,5000.901176691055,0.127613692196334,95000000,18830.83085513115,0.1224305469455894,0.125099425818281,83274637 +14012.339433670044,0.6221432685852051,5357.929572105408,3262,0,5357.929572105408,0.1275428946753251,95000000,19381.26821088791,0.1220452661052369,0.125017859667821,83274637 +14198.603212356567,0.6516408920288086,5714.501089096069,3500,0,5714.501089096069,0.1273975718032033,95000000,19924.81757092476,0.1239425728552684,0.1248380829792031,83274637 +14381.964620828629,0.6807894706726074,6070.098887205124,3717,0,6070.098887205124,0.1271452108348645,95000000,20464.478921175003,0.1252923225095345,0.1247439543640237,83274637 +14565.263352394104,0.7095613479614258,6425.994694471359,3926,0,6425.994694471359,0.1271314627067164,95000000,21004.370398521423,0.1242061938198927,0.1246761850030704,83274637 +14748.313912153244,0.7375872135162354,6782.304340839386,4167,0,6782.304340839386,0.1267709881060148,95000000,21544.441210746765,0.1216376443970131,0.1244643233851619,83274637 +14930.95549082756,0.7668135166168213,7139.600712776184,4385,0,7139.600712776184,0.1270552308603387,95000000,22085.083664894104,0.1238337018820797,0.12463919110672,83274637 +15114.357768058777,0.7977817058563232,7496.691490888596,4603,0,7496.691490888596,0.1270686893556293,95000000,22626.284920215607,0.1234566935100073,0.1245452875242503,83274637 +15297.123735904694,0.8283920288085938,7853.557901144028,4804,0,7853.557901144028,0.1266465642205489,95000000,23166.60662317276,0.1216544660197443,0.1242633380013585,83274637 +15480.888777971268,0.857440710067749,8210.377913236618,5036,0,8210.377913236618,0.1266768566536351,95000000,23707.909193515778,0.1222382430873991,0.1243029243867079,83274637 +15663.872374534609,0.8873980045318604,8566.05491733551,5244,0,8566.05491733551,0.1268844927071019,95000000,24247.26802277565,0.1215783337375928,0.1244308181581128,83274637 +15846.391675710678,0.9166655540466307,8921.71652340889,5455,0,8921.71652340889,0.1265792360459177,95000000,24786.152362585068,0.1225918546790257,0.1242333312215549,83274637 +16029.786890268326,0.9480431079864502,9278.934896945952,5681,0,9278.934896945952,0.1267234030258981,95000000,25327.473544597626,0.121070222279228,0.1243281612552211,83274637 +16213.41997885704,0.9778847694396972,9635.31711769104,5893,0,9635.31711769104,0.1265945572967529,95000000,25868.18764829636,0.1217274008347731,0.1242899091829941,83274637 +16396.634641885757,1.0087382793426514,9991.516325712204,6094,0,9991.516325712204,0.1267142726342452,95000000,26408.29709792137,0.1203969678724818,0.1243067922626198,83274637 +16579.479200601578,1.0386712551116943,10347.876336097715,6306,0,10347.876336097715,0.1265346370219983,95000000,26948.199214220047,0.1227307734831789,0.1241880603189149,83274637 +16762.955620765686,1.0695700645446775,10704.78233575821,6519,0,10704.78233575821,0.1264345738501297,95000000,27489.28637838364,0.1213983858754327,0.1241065930599481,83274637 +16946.520087957382,1.0994818210601809,11061.2769446373,6766,0,11061.2769446373,0.1264060268784372,95000000,28030.06036734581,0.1208675704827191,0.1240576386627922,83274637 +17158.9575817585,1.131810188293457,11418.61616706848,6968,0,11418.61616706848,0.126419649889173,95000000,28600.53822851181,0.12346100642106,0.1240783339262097,83274637 +17382.02194261551,1.162308931350708,11774.297981977465,7182,0,11774.297981977465,0.1261435647168611,95000000,29179.98655867577,0.1233746810954928,0.1239068020424281,83274637 +17577.117379665375,1.191917896270752,12131.47572374344,7383,0,12131.47572374344,0.1262276595552143,95000000,29732.95296025276,0.1221032197481472,0.1239043323143349,83274637 +17769.174802541733,1.2236275672912598,12488.324034929276,7592,0,12488.324034929276,0.1263172912078857,95000000,30282.56272673607,0.1207346899932191,0.1239396156478079,83274637 +17952.57149863243,1.2534687519073486,12844.588040351868,7810,0,12844.588040351868,0.1263105004013864,95000000,30822.92870140076,0.1229351218176018,0.1239536402518909,83274637 +18146.438154935837,1.283888578414917,13201.688807487488,8002,0,13201.688807487488,0.12613237252727308,95000000,31374.58929824829,0.12136180953056898,0.12385063270517058,83274637 diff --git a/logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/trial_1/events.out.tfevents.1789424879.6cdd716f9ad6.38.0.v2 b/logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/trial_1/events.out.tfevents.1789424879.6cdd716f9ad6.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..addb9002f8694f2d0cd9f1cb91a1feddfdff4579 GIT binary patch literal 57683 zcmbWAc|c9y-o{IzPKG2Ql_8WOlvIi&G88FE63vlHqmr>fp`=7arpgd9C1g&DkPuSF z5Sb-H#`kuP-(K&thtoOTfA9T1cYXHSpJ$z^ZG?p2Kb2)8Yh)!P@{LvZ-8WNi>lWnd zJx^y$sOP+pVY;@xdA&kB=LP!*dHMQ>>A3p`dFsp$@%Gj6Z2a$oox_5>Lp+0|mR5f2 z|1a&!!ICkud9SiZG^T0w z2L%7;n!LU|x!Jz{Zmz!0!6BXj|3x(Zk0Ijpz#P>f{~~7Tb#{{dQq*Eh_y4V5UQ$bg z*N4XoX}l%_|363WsPD+J*y+2J(Z7uN)&`$sPu!p~`t!7S?L%FCy**q*y!|ogcO)(7 z+GOPnvwuMndkgleh~js|Ch6NY>P3FQ}N@N=9qsuZ(q(iEc>p3$}!{(<#ls)cc1S!-`6$7)5AG< zzMG$SaPa@O&N;-}k9%?Jz7FVO`Y+VRGt^bqR6^>%`$(5Jh}Y@=hyL%s_l3A?Yb0@v zQ{J9Q81*kM{PVnhvin<6ar!(xUYGwL=j%Uvw)ec*kk~j2)WIrK?(6?cEH9e*QubzB zDv{fk2)r3u-Tky>*uM~klsC_0Z4{}H0X%J|=P;7ojO|w%l8qll51v#LY_=Sq#rjQs3x5lCG z{XYW5t(O)83MnSVnTpRy(Z;uOGf=#KPffE^F!thh8!bPe*i=tNa8ry#im37xSAZh2 zwY`8snn^KrZ!J}2R9X`#7Ivm0xG5r#BE0C^1E5e+@)S_C zWK!&$_zo#nNBuqx6gq0uG&{xr6vPNSse4sG5z>c>;HKDt6gI0>2LZ*Gt|0=7R!j=x zikG+q?cCXcK%v@`nr5e9+)-Qfl*)j@N0W-+rpQMMIlpz9KryI)w1A>DlR{@k6;dd! zSC9mXQax%~M2c1YH*Nq5Cu1sto1zLSE-1WM3lwkdlLZtqOo}tP6-Y5~a#R{n#LS|m z*(n%zRHa_r9-uJsq9V8{B=GP*GA(8gP}Ihx3n*ln6y3VsL<*}Ri=G3;;RI@$oq}Pv z%7>3Hfg&=6ir}V@M+({S&XGW|Ip?r|g2zHJ`VvwUTHC(|inXVxX?6<6d2wxxvjb3U zDWD>_DS9AlJERM2h=^B}M{8ms?Z>H^m^N zXlw3~2oyslZwM&bFeyB?9YPAv6YKW?#S~d;T11Mw_PquGg;@tGf}3I_QuI&o-T)M9 z`#ly=v}IDr@G_Aiwf0X7pqM+Dnr5e9c+qd&lEXl;(TIxRrkIQrVRO5j1&V!b-wP<( zF)7~VrXa=WF(*a>MMEcQnw^4iUP#y*#sI}LRVsp;!UZWlduvPw3e7gZ1QhL=6x)(k zAVuX1nGryd(~+8Hr(oFKsm+8!py=C;ir}UQK#Cg_GJOZ z)p#m`n<5n{K6c1Z0*c(Eu>uN3CPjJx4=Hw^4Lt=EA5y7lb_&KFrRG=C5h!wYP!Ze| zJCI_%?gM3@*l@&NK+%awVHf|3_JLcXe>GQZtV*pa?CXBDg8CkYdZN zx9@<$^opl|LWxQ7G^P+Ktfg110gBzXsA+bJpY->7zMs>*fZ|gH6~Rq$7%8^>*<=Y6 z#uq~b6rGtAL;ZFmMZV`_d}SVZjhbes_(i8s4AFWD6cx9r2yTizq$r(iEeRA_SEB_K zU6>SIZIh6~G_ckJ_QJM|nr5f?MyIISkz)W9*D9$9Zi;-QSaWS%3!r#>IdrC!cQxqe`AJ5Upfx`1ix`0BNNnvuqA1R8v z{D}pM&o$IEJH<;nMbiHBK|m4nk&57^xQP_9_xc_NilUms0*Y=-3aP^$NU=&e20zI$ ztEHydDg3>zU_}2Jv(12_;2RaeO;LdqYIZABfI{l^c>#qAlVVQNY+S;?#qr-^FW%Qu z)9e&a>3i|&`0pt|k^Y^E;HIcTioyNg<^shetzQBPRVGFC7}{P$Yg*p{igUWuG&{u` zIz?!B-Cv-%FpP@erg()E4~LBU3>3ESBggVz)2cBkavW(CKMhX31&XSAYMPzmHJxI| zsr4p6vEnxs!A((%6xV$u@zkpck{QRR=+305>ga`c)a9|)aW9M(QPb=c@97lFc4(p9 z-J_`pZi??nF*sBi-(U1tmMEa;!KCnNn1vM4H*&CP6;@Ew>=cYUDzDqS&af9}*H97M z6cVH0GjmVJDaYWBYH0skK+%&)p>ksyQY`#2pe@`{9lB7{>=cZq9kH|e98hF;ry{s1 zq>*Cagfsa5;``4vYT|EAd~R(YJpi2GZr7XgKS8!CdE zLIx@9S8D6PUKq~lK7mi6&ZPLU!3vi!K;=1plH(dcO|w%3JYRzmMGe-jKrwg$6~Rr> z1}VB69xVqH`tExK6up@g1G|jFCAj50c89%K=0i=hQ#fXnVZ>A3k~E;`8B9fRQ^+I5 zKG&EMpr{Enw&d?cA0~y1>S$boYk1~-py;!Znr5en3Mt2kje(_hK+!#lir}VDM2hIj zx4}R$^TKHXMPDXG$=4CMguc_?&IO9?SEy-rio(cHjPRC|#CJ)Nx2OniiY`cTa)xR< zppbJjvEuKA29x5M!7yAxNBz_4Kq2W%O|w%>cL~CXHXR@O!d^TJrXsj0RFJ~Q!^9mZ z$}ScPC^VTAJ`44534gwujR%T8CDb%Kh3TF>7@?$4iQmE)-Jv45DS9A!eBi zz0hV-6t)?F6xN;W4grNsFKU{dVt;rCj7SK!?+Fw?`cV=c*QEWn7mc;`}}NP9^|a8nFIif=8;c(4}u3>~0Zu2`I9IKMN>ym=upq zX%tu04yge}$s%f+oq{)>c1JCIxBC!K?2V=(xG4;g!eH=A+>6mMv6K0GF_1}Nq|zHN z&4SDW*5w#TqE~q)-vu6edWqB&EwK*o!|}eQfv?gP0UGS5=UL z=eY3+P-y5;)9e&-wZDhMTZ5Sg3?2Z*ePb$un_?tV1gDz$0Y&WIJORaECWYY)8pX4d z+4?~7_Xsu3P9YIii4@U~AASLfgD0s7Zi>-Jk+~=fHS-bISHpVf+hVl$75;HDUd6!oj(PXoom$pwOS=rJkItm=#uk9_j+ z+sR}HYMPy5{0y4iAqEq#0!42ZDuSEB3Mrm;Y-0!%+h&fP!rzM_Oo}4=j!3a6`rrfD zi_TuuG&{uv`8?c$wMUAg;kB!P!XTcS zW~Wf9U56AAmV>K-A~uN67%(XUGTR}AMM|LwP<-o7O|w(v zPQ8y5JzH%1UH2fQtaz@VG2-$J(x3C2-&-nzo5BStig}-~X}|WBvFB45F)50I0vr|0MsX>Z` z!@I49JL=d_DuSEB11V%~>iNT7SlK2DD26d9%AUz0MXLo?oq;0Rk(y?wxUYH>DW(N| z!p{c3x>6C`6myUw>#t`vP~_UHOylo`F_U8FRT-qn3bVTm6#HjU)9e&F)8`Edh!Y3o3$} zVlh(4me246iqVtq2`EM~DSW+XcAp5n6a*A94%9R|g<-HQQY`x{vj!-RI#Utc6cI=< zSj{?uG2!R(K&T+a%#$)D%=frQ#?6Jd*7X75!D(f3O7;_+!V1$;a70Q z5%!|8)pr5KXeNbA7|n~jm-^HLMYs0UG&{vG3j^GXt;>SH0>yeIDuSD0B~l#Nw&68U z#D%SJ;_t;6CWVdyjbfOY%x<8#xQv=+r#Q4$0V%HDFxUqa=T=Y=+!S$05wZ9+{`Nqh zr!bRGVZo%ZdqBIRR^@EQQ*WXIHO)@Z@p31mxHbGBerw-LnTp`1SdA28kMGlhy;%5j zlYn9@lR|TTOI(7?@|pOpeVWwf|EAd~jPEbOhz#j#U12XQc`D56*D z%;N9GI3~rL5ZWDe=}(dq+)*-XscCkK8Le(0MGjB<0#GbSr6RZ~l91wQ)r@;U(X-bn z0mXPG#SabI9d)@#A77bY)}p4_DIU^)d$34iwgONn=ur{e6v;@jtnUK+?ZFAxNzVMe zn82h6kf-g1^f&Vw*bDW!)HFMV-R_sT7pJAKnF2*(5Ea2qu^uU2X-&eQw& zDWnu>XZK1IQ~czAz-wxnogzku_66s~6%n35@xGRd;HKDs6u(=l$G{yGTIk`z-wP`y zg?1Yn#g2@F+OQXSCDb%KMfUs_xEI@0)5ieC`8!kuH$^H^#Omx$0gBwDKLU!0Op2lN zX%wYNuNMNvpHym^ouZ)CUZjvbef|YdNTyQ}+!ULUVy^3o-$3zm##&eYUQA+AY&bxB z9ThyYe`}!d_n@ZPDca4dK#H$E+dY87z?X{Prr3fM%LYEfZ;F?@_jKb^STiZ^e4tT` z9Bv*B6s|thG&_Z9$Q`6`%{8$H3W;DUf}3I+Qrwn#^A`4^R_UOCVltCrZUgPqGmnqz z1r$HksA+ZzHKh$mk#cVf{sri?zElJ^#SWw>_tD4#icOnM-1&Q9!=%`FgSHo1(=HtY zibWaJG&@DB&$N#h?bhrY4trs+kBZ=?$Uq9)dAb%r5noU!ps;09l%+}Ig-oAbjen({ zdzG4Ir-*xazy!WaI%)B?5-7gjq9V8{b|S@~*FpHL{TF|05B^?EVN$fop;0Wp_69#~ zUbBFjW~Vs&Wk0^xlP?inF_lSSdPEBEsN>tp@Y~6) zTd8SwigycoVMPDd&i#SHW)~H~O|cIteymR)0~E*0iP38~y^t_36|!J4N>hU8I;fF6AIl zta7CyxGD0GV%#?GE zLq%{?oI;A_2i`0Lic{VubNPEQi%BttN4ujo-Bgi+y;v7WO|w&EJe-Srp`H_)2NWa1 zsR(Y0Ge|L9Rc8TEeC_*OK;g`!a2i6pqY6H~ehd_nL#SzXipTWNKz=Od$pghtV=981 z;v7<})Hg~83Mq3xAO2prFe$w5)81c9xVRgCiK<{pO|w(HqEp!1Kd=NS;%%u2Zi;-Q zc<|I3e{nZ3vz0HO!j(x;r%Kz4;*m4(lbl}rscCkKI~gVD#TuKi~XHsOT(mLwbZd?3o+*$3YX?BX_!?Yj7^lq7-02FhTs0eO~OGt6SEj11( zg1=@6C_I=HHZ`;lljarJjR3nP{!-KI6vI{&VMo0%vWN$Y%vPEIEr6S%7%2iTzQMOz zr_CoE>CyN+!}FXEH0-HM+Y%oK-3#!Tw^il>YJh1Wa43fyd(u`*sR@BkN`EhXVACrF%pDR9*X)Al$jbw=VP{RjX65I#<%Sx%g z=`0)OO|qCzHN4*;&m7+2=e}6o8Xg|`#GlLyd92|Mhne`uz@Bx{MHc@S5~P1}m#lOs z2@>_WhK%Js@HkyI@?WZE=PxO;ix-pBrXOs0eY6Fx(q86$iTS_0BQbq;$W}*@JdvJk z7$JE9o4Daw)T^QYBCV&b{wS*yML31r-Kpa(7}gWZY~7}7xPGpcn;c(>M|dmH5=Ax>KML*UsxV$d3F=+ zgpj{0n+$jOeKj)9ex7GvKZw0_@_^kMxF)mvk_^skBJl&SNri!mKD;V;WY#iVy|GL# zNpqFSx>K|Y__X379_YcABt%%5_&$=vjm|1u;{#=)WlNHpRVHIITj;~3KI1?V*~?@> zJnaQbuB==Pl*s^Rk|Ac9JOumJY)TYbi)scKnz+l!6*u5)1K~d^oz@3d&?a-`V~8HiEow%~dAp?P&*kUY`fF1AQO~ z5mqMeGMSUm}xzQv;%rbFQ zUuXHL1qG$@mcZX{2nGLebjjR$($oD!T zezJG_Rp8*HF(j#34$3KvWRE zbTHGb72cHUe&%h#!9r&;FT}xRb?WHg^U@k;aB!C=2@=&o#`5+g?Mnm)WqnC%(+)~n zx4=~%ay|D99P|t#c_KMjHRCusIO>U!12`xZP9ixSblHdwF1y=)8aUV@f~1J&plMHk z{7UZKImsKEjSjAr<8rX&>YsS5*FHbv3=Ya%AtAyXtcs#J=-<|KE;v|tgCsS}L3Mfk z;ozWuIZ0%9Fy}VyoR`07RRIpFKOz}oIyli<#~B=KTTQZ<4tlIoK?gTQjJydB#=If( zLLBU@YlL@qq{L`_aBy}l2@=&o#_~*y-)Vq@@4k}MrXBp!&;eKJH>c-$bnrLH6Uo7V z@IrL3b^bC=9=CR45#KjdK;YIb*z4bC$ z{C5X!xXNVwa@v8O>*jC>%H&WE2@zH%9lU5|vhu^=VNfQWPm!c%m5Fr1qit}hlk!O- zdzn;xpp{AYQ5`2knf$s$GQ=#C(y|F9P$o;RlPqSLSeNKxnV8Gn(}yzIcbm)$DHHo! zR;a0a%}^I8li!sjNYpZ6EH80Oy%dzmiYk)YbeZJ0?18J?blqzQlu6!8k|$D`4AQ)f zWiq+l+*u$_-Fp(rDUQT-6sO?(O&cha@n1-acx4iGC;TEf_<4=Ljdr7hSK4tos8CI- zl<~S|+2G*dR1zZ0!S9YV2m8E;9S;t!+)k33<>33Zk0yYFzjl#Cb_ZA9qB;0(|F~js zFf*HEi0Pn9P}wJNaMV$f#dNU0(s*?6>jRA(aPZL6oz-@Ut5f`jIlNNUp#_WQ1btF(&^xq}XtkUWtbOxpDn9UR`#U^+N>{x*r^bg;*3 zbnvoq9~W>?<333d&p}1Mmc!sc?=&@ZA>YB9EdK^cFZu6y7;hbF&>QaX3C?7k{XlI;5KwrHY~6Yoaq!G8f7uT}Ng~|Hd*oC(l(TM+g5!9J9~;Eul|anO158O$v&zJ@;BPft>fK{VB72$GUZa&sVn2hOP$rkHNQRhY zQg+T>9m?dpEy-e*iMp3FmdWiG4Y5!rfz!#nkTO}K5Q3U^s=qfE%4C8w2@hBp>)vQ=BR(+=u+N z+lN7!j0z_y;+0AG!#k1S;Ms&DaRVA1+@rwdpz=Z5fqwJbwdvrX)J75_%)!L=GzUWz z`Yr_r8@7_9W;y8aZq`X~P;VDWWOs1DLYjk%-ULL0gD3Zs3^5)2Wt+4M9PF1vvRE8E zJRkSocfoM{hlmsM$h;5-V?(3x?w(d|x)>asdX5B%>>z!4^%3u{f`bl4B(-S=hq{c# zRTj5-;Q*o!4pg-s3D?ypmxsVWHW%AjQc42A^%U42~Oq@fK znpGxJy*nR-OWnbrB(j%@3jJ%nKNAMsfih7HB^hFt$z=B;b10McOGp;8OkUT=;7x3o zIyM0gbdzW@FQiOvny*4lr-YpT0%fANiUf&TCXD56N>;anGAT+VsZE!Oinleca_O9_ zbxcDWOH~LAkOivB$88{{2sUuGOnHmP$p_SNs4%7l9hYY4G#2s zR;uy?8y&3c#N}X=741M@$=l-t4nCerLWDUurH~01+&_>E&TA53jMpSFpbGy;Q*Twuc0Nc+XRb1N zeC7up=y`h;*F%{kCzB9iWg@Rkdv`FraQ_h~6Q9i_saa)m}lq%!$c&>73*R)_4K zAkNWJ63Hn}z6tJw@tb~nP$n(zk`(dEL}!>h{>j+Mm4o{Y;(tokmD9mG+KX!Uqw}_a zgXYO3M3{qzq-hS$_V6?U2c0*Oq-HrNr+K{$9CX}464@P0+e~xN?Sw%UI5=(($q>`Q z%^xiS!NJc5NEXw<0RKI>_xD=2V1Pq z!M#88Ej}C^46Y>eLLAK8n}&Dyt%7fX;NZX~BuG>T8OxhDdU`cD825stHtpct`!jKs z(#hux!ND_cNuEd!*52xi4i<+U(F6x~*O5q02lE|qA7-aaxdsl}G>{bW9PC`KoC7ba z+ejHw4))}A=Slne2M3Sh$w(?l^pf7@NIL=b{EH>w4xcSY#@Sy~E9rNjy^?*b;04#D zvjWKw`I@}Fr7;o8MDN!hgTajldW!pfu|oK_|`e%DAs znf#C=NzE#gxU(O8;8G9jND|r0WQH!SOkPN?83kpM)svuW#r04wkENIe5$Y57xn4*PZL&K(}o5oeB}=;4pidgICR-RfB`hZAntI985oG zxf>k(-H{}+JD9ISbFi;{)CF*Gr!vV9)4{VY5$nLgzdcA6)4}>1#klv&W{kf94nEZ& z^Fkav*JUT(-LKkb-T((b4In|HI>=a_UGCMl;GppklG?O`?NZ!vm8)-jaRLXo4reX~%jvbb3*ZiyRVL%?2YRWiHX3~8dR+}%lM%g02In=AScvzga>O?LVe0NpzsBe` z9_W{QaFvOsD(&@2#>~8pP$u^>NQkg9(Ws%-(lE`3^-v}?dr4BW%A{~*!&SJ{k%vhl zdzqNsr2P$=taDU`{evm}dICjND|al=l#S4TscC>N4>A!YJx zau#apW>9_=%H;7C5+rJwFqY?=|9m2piONlq+H{$i1$yBsyJnt@fig)iCwU^3$?`H2 zEEDaE122O(L61lzr#OaDxDP2Rk$9lre@;@wE0d^o8TcT+*=2pM^Bqj@#pPh^7TP=X zdow0GfP=RmkPu-G-W^3d(08TfJAs2`&qz|U9DLSeS5I)T-)oY{?qH!Z?Lb$bKB^Nq z*rS$Yi0R;?0~0oYgI-@r7Sq8}l_$9OgZg%G00$F)lX)QyYR6^c-EDuTay2*@DAkwR zc~Ko?EKhCJuDjsi23eBYw1Z`q-nh!eo3w+#K{I)hCz6BJ6Gx+iPN`StfP<2qNhGI( zGOKVOT3Z))g9BYgjiiX@;7)DzFgVcLMLlcaJGfq*`FOP-XW!#t96zvL1MYCOcrwm@ zpg)>gieGz8ZF|Wcu8Czb$>6*uJ67X0iQBX90K7ZU$_}#AYdp|@s&kdeqjK7}cw?5P zwSh93aGZn)E0fI~Xk~IRa(o1oN%~om)T}a*Z0`~YmwI#|Nn|e*OEp@V>>2VQ3Cbk! z8p#l|Ove5AlMZF_=N8FgmdVO7Z*ap(E$nNdOx!BSypS?UE}cu|qEqXL%ioA|wZZgFKxT5BLri_Th5yM<3dYYHgX_H^D(Q0}>+4!OKf% zFRDY`=9qzlpNEsAW;yusWA%G*aP}CI$nId~A=-=TBa;WN1_vKnk_<5&tVn&W1P->f zC0R@d!+h&;@6~sYN&^S4JCJ!H4jTN<#k;#q;qe1F&=Y2nAWh%FSNS98?MRF zY?8rwO-9mw7GYa2aSqBvG4;M1A0%6Yt4!n^Xuo4AUG#l0lu7*#5+bZjRwmQRWM;%S zM<|nCStO}hWfB%zh<~MvRmtB;pv!5VK6)KkIW2%Ea&t$zqmC z^mqxh>%7x(6)2M<1!P`GnUoov#2~K+b5=l^oGm6nqLv9`d75i}YC@R|zClu(E|WFS z196q(DrC!{Od{@*Jdw)8Byt9piD?g2ClKezLlVg;j>}doli_#v--0sf@SLQGS0;|b zUAlvVO3RPD<~#UHlgmN*KeP}1B(nFu1P8McNQf{8H{7K;D7h^n3>;KRAxX`0a7TEV zCDdI+8cAe#u*Cb?f^c;Goel5+tgFjOBIJyN(BX%4w3?w1Z2|)BYw+%ehSjIJm5U*c}|~=uWbj4nFphLv{M-53>gcZ+Me=Ar4O3c>#lZr?or`4qosl zL83aySYCcWeJz}WJwr)q(+;lP6o#whjqf)J9Nf2<b|+?VEu@#AK@H46-^>J z9bB*v9UL#SsstRI8AnpYbI?9EH4Z){n<87jkni9xE#`~r2{UOgs)J`(m%|-Cz5^L& ze^Gtbb2YxGUZEfN9j?i`E+m8VnpEw_YclHB?qDdB_Q%hCzjUM9sCX=T!RTFomclL@y; zhL~lN=l0+Vl*yt>lEo~Ovp?n0u98;={y>>XR*`ujWm5OJ2!qbv&X9pJ$$LS9L@g7> z^18IluYxl9`Ie+MT_%0Q7UC+MmOa@EWnx=L@+5z7ag1wK|+K%cr2dg z;HJ@6UxR~JSCFJ;IrvFuj14$gx`rgOJ1Ch!bFll)R3mWkdJ4%9)4|yFi=Tjlnp;Q~ z)4?$bN_bPYv|K(79JI+K^FkbS>2ei=7ClT40|%q_k|0qXWGwHIWGDRU`Pm_o+O&fQ zr!K)rO0S9LsCwU?{m}S2Z9aOK@y8#ZaK1(7w9n{Q22k%|8oC6NlTqG&tIoLHb z%o7guG`s72`3~0VFc0)C%V;Oy#>d<8;SPT~lZ>+;=(~TM#RL6TklPHnCO14u2In=2 zIE~lDvftR+&V$Sn?1qb@wQe$X+Jy7ieWNXRc%_l*yu%Bty(H@!w>39Li*VBFSQwNe49* zwCiZ6?)WL$n-nrHq)bvLmtv4_fs{Oy$<{OyBx;#3miOtn)n7Q!Q!_|v(`7QIYXq+H zh3*vxD3hyMBu}I=u~&=4GLg}>mI84;93qjN;sl&Sao$?3bAvK5J5Ex>D-*pay_Vph zj^X>&d7Z8v4)!x6A;KKAwx>Cm5;q%vOEzm9NotmZ^Jf>|2M4Ps zkwkU}t&(XDns#!W1rEkdB^hEm_$A^%9yloPM6#F;Mjq;kH)Xg&{BAhVBizWm5C^?h zmSIq&);Sw+P|2GFiRvI@d8_^=jsyo+_>NIQxOFcRvLW^n<#Qzu=loT1YZDuSt(<*h{zDIX;0hS#(-g(XjDA|E9}T zCVAUwAGxQ#JrNIOl2u4Tgq6uAZCaVs%Ffvd2f9-UNorP^NZFQOflDoSnr)714GO>I~=7p5Wx`J{H3L1In zK9tF}cO*#EGGQ$5+R{2>D3echB(>=>d0kI?1l8fNzsul2AK5_iL@JZ)nw3~4y~jrX z1aXWd^r%~&Q=B}SI9aMI@gw)}RwPBdGV#-ONdgCVyLqJX9sDqa%R!%2wD$;;tIx%N zg9Sb$M3{pgI?x>aF8@Ie9MlOWNzHQ5rzUGVI9L}>64@P037|Q+-D6`OIOr2WGQ@N+ ze!5gWIQS}-WHB97x!WJr8Mg97066HBK<0%w*!|ai40L6ozx28!> z1qUTFNovy$@;sO0DqU`D=mQS!+)MIAa ztBW^fcKge-z`;*#$-EE;6$77OkiiM7Z{Xm$jwDD_2N}!jJU4kTIM`O1q&Dr~o3^pI z$~i%YGQh#O9wbjB2Q{B0p@Z3H=T(4%B^o4>)4}{od;yuTeMvkxxL1dyi09x@l@BrS zqT1rvy#ag&D~2*(RNrq!`|;JG=bl^O4sUgijI+O}RyR6m4F5LMyF;Bj!ZrC=L^8Oq z$$h*gtIzY6L7A+Kd2)&mvc{0BOdjp0mC5+}TKpZ0?ivyztV}8kXn(Wvt7fAyl*z3W zlGLm+S$tr^9k|rNTSy{%nOxSUmC2%AcOOBSRArD1G0SB6A9wthfBp87EM}PuZZJT* zyk;)C17-5}5SbTJCJo=7V^BlQQadP<{>Mp>sAa-f-VL*8ER&08NNUq%B3H5kS2-X>1T2l_oDE(gsz&?+V7uk#Q%(0i>UA;KJ7bAjgI?K7D}!NIgdlGH2*2S1oP3mjD3 zKoZ#t*dF^Fkba zHU1R_mCH_i1rBB&B0-`$$XH(J>7!n7pnp9^Qk!vEwLKFYTv|jTIUUSDhkxy*KZ|5=UK1DE9}oRmX@WlvPxV<|$_L3Z z<|>nXbJ{!fm^HIIL74>3Cn3Vhq}?4_nFPh`ZwF;Ecri(8R+%`AbTx)cogPIJ*~`Rs zI;~9FCe*itGErDXGQ=#CsIQ^;a3`HalEo|&tDffA0ht|)^58(Z5BsI&yzy}3=!NHU-B$3_0xM?&8Gm|dVfrDFq zk_<5&bUT&l3kUi}sV&q6XFBMZGmdssIvWfD2S@S9ybuQ;T(8BTu}jU>z(GxU5+tgF zjOBgVyL>JAgHtk>^v(>oDUuTrOz`-IFk|&ab`vP~MgYq?U_#@1h>LilWLHRG} z;P!9V`+|c;S|mk02k%;h;@4hdCiouYJD5L$d7zK)QilintDXnd;11WDLdMw-^ns(V z;yZNP_7CviT1s;y8JyQd;s;(6qYqv+P$nm*#6IDJ6pY|1lepEiuf6WxZGlf~-0Va` zgq6wcU9>WZG)=&VJKb?7NzE#g{rhw@;ZkeQC5h~1QlUvJlaCG;M?;wu1&|Ce%j9rM zmwYIbZ=ocMStb`OCZS#L^S57wGWoHD%nK=>sVYpsRo+W4oDOA@n?&+NDw9zAomeL8rOV~vKrh@#B00s8r-^emv}+EO ziP<)iB3_v+GD%(ww}g{jfuTv`fu1;0i+M{_S4_ccq&AB;4}kP%k$Fxa6WX5>~Ik`(cPEc(tF_J0IA=(qp? literal 0 HcmV?d00001 diff --git a/logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/trial_1/flags_0.json b/logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/trial_1/flags_0.json new file mode 100644 index 00000000..c23518df --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/trial_1/flags_0.json @@ -0,0 +1,62 @@ +{ + "logtostderr": false, + "alsologtostderr": false, + "log_dir": "", + "v": 0, + "verbosity": 0, + "logger_levels": {}, + "stderrthreshold": "fatal", + "showprefixforinfo": true, + "run_with_pdb": false, + "pdb_post_mortem": false, + "pdb": false, + "run_with_profiling": false, + "profile_file": null, + "use_cprofile_for_profiling": true, + "only_check_args": false, + "op_conversion_fallback_to_while_loop": true, + "delta_threshold": 0.5, + "tt_check_filter": false, + "tt_single_core_summaries": false, + "runtime_oom_exit": true, + "hbm_oom_exit": true, + "test_srcdir": "", + "test_tmpdir": "/tmp/absl_testing", + "test_random_seed": 301, + "test_randomize_ordering_seed": "", + "xml_output_file": "", + "experimental_orbax_use_distributed_process_id": false, + "experimental_orbax_use_distributed_barrier": false, + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "workload": "criteo1tb", + "tuning_ruleset": "self", + "tuning_search_space": null, + "num_tuning_trials": 1, + "data_dir": "/data/criteo1tb", + "imagenet_v2_data_dir": null, + "librispeech_tokenizer_vocab_path": "", + "framework": "pytorch", + "torch_compile": true, + "experiment_dir": "/experiment_runs", + "experiment_name": "submissions_a100/ademamix_golden/study_2", + "save_checkpoints": false, + "save_intermediate_checkpoints": true, + "resume_last_run": null, + "append_timestamp": false, + "use_wandb": false, + "profile": false, + "max_global_steps": null, + "overwrite": true, + "hparam_start_index": null, + "hparam_end_index": null, + "rng_seed": -826981189, + "set_pytorch_max_split_size": false, + "pytorch_eval_num_workers": 0, + "capture_jax_trace": false, + "skip_evals": false, + "?": false, + "help": false, + "helpshort": false, + "helpfull": false, + "helpxml": false +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/trial_1/measurements.csv b/logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/trial_1/measurements.csv new file mode 100644 index 00000000..75532982 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/trial_1/measurements.csv @@ -0,0 +1,157 @@ +global_step,grad_norm,loss,train/loss,validation/loss,validation/num_examples,test/loss,test/num_examples,score,total_duration,accumulated_submission_time,accumulated_eval_time,accumulated_logging_time,preemption_count +0,0.4999999403953552,1.9294967651367188,,,,,,,,,,, +1,,,1.9395356163303377,1.9414668747834607,83274637.0,1.9351263324861223,95000000.0,11.226190328598022,2084.472838163376,11.226190328598022,2072.9244389534,0.0,0.0 +1,0.4999999701976776,1.9290478229522705,,,,,,,,,,, +2,0.4999999403953552,1.921044945716858,,,,,,,,,,, +3,0.4999999701976776,1.9050517082214355,,,,,,,,,,, +4,0.4999999105930328,1.8801676034927368,,,,,,,,,,, +5,0.5,1.846119403839112,,,,,,,,,,, +6,0.4999999701976776,1.8034738302230835,,,,,,,,,,, +7,0.4999999701976776,1.7527825832366943,,,,,,,,,,, +8,0.5,1.6930155754089355,,,,,,,,,,, +9,0.4999999105930328,1.6257661581039429,,,,,,,,,,, +10,0.4999999403953552,1.548953413963318,,,,,,,,,,, +11,0.4999999403953552,1.4650793075561523,,,,,,,,,,, +12,0.4999999403953552,1.372021198272705,,,,,,,,,,, +13,0.4999999105930328,1.2737436294555664,,,,,,,,,,, +14,0.4999999701976776,1.169583797454834,,,,,,,,,,, +15,0.4999999403953552,1.059100866317749,,,,,,,,,,, +16,0.4999999701976776,0.9477307200431824,,,,,,,,,,, +17,0.4999999105930328,0.8345035314559937,,,,,,,,,,, +18,0.4999999105930328,0.7243095636367798,,,,,,,,,,, +19,0.4999999403953552,0.6361420750617981,,,,,,,,,,, +20,0.4999999105930328,0.5407810807228088,,,,,,,,,,, +21,0.4999999403953552,0.4541879892349243,,,,,,,,,,, +22,0.499999850988388,0.3838339149951935,,,,,,,,,,, +23,0.4999998807907104,0.3250519633293152,,,,,,,,,,, +24,0.4999997317790985,0.2818763554096222,,,,,,,,,,, +25,0.4999996423721313,0.250862181186676,,,,,,,,,,, +26,0.4999992251396179,0.2341355681419372,,,,,,,,,,, +27,0.3933596312999725,0.2233276814222335,,,,,,,,,,, +28,0.499999076128006,0.2261368036270141,,,,,,,,,,, +29,0.4999993443489074,0.2229228615760803,,,,,,,,,,, +30,0.4999993145465851,0.2230022251605987,,,,,,,,,,, +31,0.4999994337558746,0.2246851921081543,,,,,,,,,,, +32,0.4999993443489074,0.2177482545375824,,,,,,,,,,, +33,0.4999992251396179,0.2107509970664978,,,,,,,,,,, +34,0.3660013973712921,0.2029299885034561,,,,,,,,,,, +35,0.2790789604187011,0.1997946798801422,,,,,,,,,,, +36,0.4113948941230774,0.1954047530889511,,,,,,,,,,, +37,0.4188576340675354,0.1915183067321777,,,,,,,,,,, +38,0.2407293915748596,0.1864883303642273,,,,,,,,,,, +39,0.2366956621408462,0.1818538010120391,,,,,,,,,,, +40,0.3274478614330292,0.175271674990654,,,,,,,,,,, +41,0.4169969856739044,0.1781215071678161,,,,,,,,,,, +42,0.2517755627632141,0.1722897589206695,,,,,,,,,,, +43,0.1420656889677047,0.1689940094947815,,,,,,,,,,, +44,0.2427321374416351,0.1665253490209579,,,,,,,,,,, +45,0.1800223141908645,0.1636734008789062,,,,,,,,,,, +46,0.1418801844120025,0.1633295714855194,,,,,,,,,,, +47,0.1995927095413208,0.1602639108896255,,,,,,,,,,, +48,0.1239930018782615,0.153117761015892,,,,,,,,,,, +49,0.1061840057373046,0.1526917219161987,,,,,,,,,,, +50,0.1044609472155571,0.1485991030931472,,,,,,,,,,, +51,0.141949787735939,0.147986352443695,,,,,,,,,,, +52,0.0898748189210891,0.1438856869935989,,,,,,,,,,, +53,0.0972306728363037,0.1411078572273254,,,,,,,,,,, +54,0.0722352713346481,0.1402346640825271,,,,,,,,,,, +55,0.1066087782382965,0.1392953395843505,,,,,,,,,,, +56,0.0651723891496658,0.1385523676872253,,,,,,,,,,, +57,0.1128823533654213,0.1361799985170364,,,,,,,,,,, +58,0.1016194373369216,0.1346543133258819,,,,,,,,,,, +59,0.0567256733775138,0.133585512638092,,,,,,,,,,, +60,0.1324920505285263,0.1319817304611206,,,,,,,,,,, +61,0.2161526381969452,0.1343812346458435,,,,,,,,,,, +62,0.2977602183818817,0.1322636306285858,,,,,,,,,,, +63,0.3189073204994201,0.132511556148529,,,,,,,,,,, +64,0.2055176496505737,0.1332032084465026,,,,,,,,,,, +65,0.0357300341129303,0.1320581287145614,,,,,,,,,,, +66,0.1558267772197723,0.131263867020607,,,,,,,,,,, +67,0.1999049037694931,0.1342628449201584,,,,,,,,,,, +68,0.178978830575943,0.1332678794860839,,,,,,,,,,, +69,0.1431338638067245,0.1328818500041961,,,,,,,,,,, +70,0.146267220377922,0.1346257328987121,,,,,,,,,,, +71,0.2479027956724167,0.1303297281265258,,,,,,,,,,, +72,0.4043453633785248,0.1332584917545318,,,,,,,,,,, +73,0.499998927116394,0.1339415609836578,,,,,,,,,,, +74,0.450487732887268,0.1328752040863037,,,,,,,,,,, +75,0.1857243329286575,0.131912276148796,,,,,,,,,,, +76,0.1274020075798034,0.1318823546171188,,,,,,,,,,, +77,0.3301296830177307,0.1322830617427826,,,,,,,,,,, +78,0.4225925803184509,0.1315215528011322,,,,,,,,,,, +79,0.4149493873119354,0.1317704319953918,,,,,,,,,,, +80,0.2931374907493591,0.1308342516422271,,,,,,,,,,, +81,0.139598935842514,0.1316512674093246,,,,,,,,,,, +82,0.0212233252823352,0.1295006573200225,,,,,,,,,,, +83,0.083962894976139,0.1295854449272155,,,,,,,,,,, +84,0.1389927268028259,0.129141092300415,,,,,,,,,,, +85,0.1586644053459167,0.1286058276891708,,,,,,,,,,, +86,0.2633544504642486,0.1314536035060882,,,,,,,,,,, +87,0.499999076128006,0.1314496845006942,,,,,,,,,,, +88,0.4999992251396179,0.131742313504219,,,,,,,,,,, +89,0.0280385147780179,0.1303292512893676,,,,,,,,,,, +90,0.4999994337558746,0.1322777867317199,,,,,,,,,,, +91,0.4437057971954345,0.1309780180454254,,,,,,,,,,, +92,0.174316868185997,0.1282465755939483,,,,,,,,,,, +93,0.041129320859909,0.128909021615982,,,,,,,,,,, +94,0.2293572723865509,0.1308985501527786,,,,,,,,,,, +95,0.4999989867210388,0.1281294226646423,,,,,,,,,,, +96,0.4999992847442627,0.130715936422348,,,,,,,,,,, +97,0.0999199450016021,0.128043919801712,,,,,,,,,,, +98,0.4999995529651642,0.1327374130487442,,,,,,,,,,, +99,0.1403219997882843,0.1290193796157837,,,,,,,,,,, +100,0.3857973217964172,0.127835527062416,,,,,,,,,,, +191,,,0.1290974709218704,0.1299837552751864,83274637.0,0.1327623055301064,95000000.0,368.1394748687744,3541.5391387939453,368.1394748687744,3172.300148487091,0.0942411422729492,0.0 +381,,,0.1258799900329567,0.1274798618422865,83274637.0,0.1299994383993048,95000000.0,724.754016160965,4979.114598989487,724.754016160965,4252.43236041069,0.239152193069458,0.0 +500,0.0853532254695892,0.1324224025011062,,,,,,,,,,, +582,,,0.1276648518342261,0.1268212165814573,83274637.0,0.1294382748072574,95000000.0,1080.6785175800323,6388.577245473862,1080.6785175800323,5305.238393068314,0.2690329551696777,0.0 +799,,,0.1268228745132321,0.1262535467938525,83274637.0,0.1286774359707481,95000000.0,1436.580223083496,7744.562363862991,1436.580223083496,6304.588199138641,0.2972466945648193,0.0 +1000,0.0104395793750882,0.121035985648632,,,,,,,,,,, +1005,,,0.1270127466420078,0.1260220589630651,83274637.0,0.1283416939750269,95000000.0,1792.2344620227814,9142.104288339617,1792.2344620227814,7345.77540397644,0.3252794742584228,0.0 +1211,,,0.1238404780887573,0.125671887563222,83274637.0,0.128079115155993,95000000.0,2148.102383852005,10492.624114513395,2148.102383852005,8339.750897169113,0.3561697006225586,0.0 +1425,,,0.1228256604923534,0.1254992503295916,83274637.0,0.1277606746392501,95000000.0,2504.586869239807,11852.522161483765,2504.586869239807,9342.487866640093,0.3837389945983886,0.0 +1500,0.0444962605834007,0.1232394129037857,,,,,,,,,,, +1640,,,0.1237874689742632,0.1253893478867979,83274637.0,0.1278642458468788,95000000.0,2860.7693309783936,13108.333644151688,2860.7693309783936,10241.437473535538,0.4124159812927246,0.0 +1865,,,0.1234818861110547,0.125347017459352,83274637.0,0.1277524258140162,95000000.0,3217.3857910633087,14422.237087965012,3217.3857910633087,11198.039835691452,0.4407162666320801,0.0 +2000,0.0298742670565843,0.1211245208978653,,,,,,,,,,, +2110,,,0.124606515527887,0.1252922273175991,83274637.0,0.1277287235606946,95000000.0,3574.3835921287537,15620.62188577652,3574.3835921287537,12038.732187271118,0.4701969623565674,0.0 +2343,,,0.1233789449432704,0.1250593557207049,83274637.0,0.1274191955836245,95000000.0,3931.008520126343,16699.2622692585,3931.008520126343,12760.052886724472,0.4996538162231445,0.0 +2500,0.0113500887528061,0.1248226091265678,,,,,,,,,,, +2572,,,0.1220150500905709,0.1246565797599177,83274637.0,0.127010758220793,95000000.0,4286.738893270493,17569.906219482422,4286.738893270493,13274.276980161669,0.5351388454437256,0.0 +2802,,,0.1231927992140252,0.1248939614144185,83274637.0,0.127454749205258,95000000.0,4643.671773433685,18256.95265340805,4643.671773433685,13603.696100711824,0.5641770362854004,0.0 +3000,0.0185175463557243,0.1230842769145965,,,,,,,,,,, +3020,,,0.1224305469455894,0.125099425818281,83274637.0,0.127613692196334,95000000.0,5000.901176691055,18830.83085513115,5000.901176691055,13819.645005702972,0.5928189754486084,0.0 +3262,,,0.1220452661052369,0.125017859667821,83274637.0,0.1275428946753251,95000000.0,5357.929572105408,19381.26821088791,5357.929572105408,14012.339433670044,0.6221432685852051,0.0 +3500,,,0.1239425728552684,0.1248380829792031,83274637.0,0.1273975718032033,95000000.0,5714.501089096069,19924.81757092476,5714.501089096069,14198.603212356567,0.6516408920288086,0.0 +3500,0.0354510173201561,0.1217548698186874,,,,,,,,,,, +3717,,,0.1252923225095345,0.1247439543640237,83274637.0,0.1271452108348645,95000000.0,6070.098887205124,20464.478921175003,6070.098887205124,14381.964620828629,0.6807894706726074,0.0 +3926,,,0.1242061938198927,0.1246761850030704,83274637.0,0.1271314627067164,95000000.0,6425.994694471359,21004.370398521423,6425.994694471359,14565.263352394104,0.7095613479614258,0.0 +4000,0.0418413206934928,0.1211787015199661,,,,,,,,,,, +4167,,,0.1216376443970131,0.1244643233851619,83274637.0,0.1267709881060148,95000000.0,6782.304340839386,21544.441210746765,6782.304340839386,14748.313912153244,0.7375872135162354,0.0 +4385,,,0.1238337018820797,0.12463919110672,83274637.0,0.1270552308603387,95000000.0,7139.600712776184,22085.083664894104,7139.600712776184,14930.95549082756,0.7668135166168213,0.0 +4500,0.0251874271780252,0.1160850077867508,,,,,,,,,,, +4603,,,0.1234566935100073,0.1245452875242503,83274637.0,0.1270686893556293,95000000.0,7496.691490888596,22626.284920215607,7496.691490888596,15114.357768058777,0.7977817058563232,0.0 +4804,,,0.1216544660197443,0.1242633380013585,83274637.0,0.1266465642205489,95000000.0,7853.557901144028,23166.60662317276,7853.557901144028,15297.123735904694,0.8283920288085938,0.0 +5000,0.0292757228016853,0.1210534572601318,,,,,,,,,,, +5036,,,0.1222382430873991,0.1243029243867079,83274637.0,0.1266768566536351,95000000.0,8210.377913236618,23707.909193515778,8210.377913236618,15480.888777971268,0.857440710067749,0.0 +5244,,,0.1215783337375928,0.1244308181581128,83274637.0,0.1268844927071019,95000000.0,8566.05491733551,24247.26802277565,8566.05491733551,15663.872374534609,0.8873980045318604,0.0 +5455,,,0.1225918546790257,0.1242333312215549,83274637.0,0.1265792360459177,95000000.0,8921.71652340889,24786.152362585068,8921.71652340889,15846.391675710678,0.9166655540466307,0.0 +5500,0.0171383563429117,0.1207834035158157,,,,,,,,,,, +5681,,,0.121070222279228,0.1243281612552211,83274637.0,0.1267234030258981,95000000.0,9278.934896945952,25327.473544597626,9278.934896945952,16029.786890268326,0.9480431079864502,0.0 +5893,,,0.1217274008347731,0.1242899091829941,83274637.0,0.1265945572967529,95000000.0,9635.31711769104,25868.18764829636,9635.31711769104,16213.41997885704,0.9778847694396972,0.0 +6000,0.0231908075511455,0.1208488643169403,,,,,,,,,,, +6094,,,0.1203969678724818,0.1243067922626198,83274637.0,0.1267142726342452,95000000.0,9991.516325712204,26408.29709792137,9991.516325712204,16396.634641885757,1.0087382793426514,0.0 +6306,,,0.1227307734831789,0.1241880603189149,83274637.0,0.1265346370219983,95000000.0,10347.876336097715,26948.199214220047,10347.876336097715,16579.479200601578,1.0386712551116943,0.0 +6500,0.0182925220578908,0.1156271621584892,,,,,,,,,,, +6519,,,0.1213983858754327,0.1241065930599481,83274637.0,0.1264345738501297,95000000.0,10704.78233575821,27489.28637838364,10704.78233575821,16762.955620765686,1.0695700645446775,0.0 +6766,,,0.1208675704827191,0.1240576386627922,83274637.0,0.1264060268784372,95000000.0,11061.2769446373,28030.06036734581,11061.2769446373,16946.520087957382,1.0994818210601809,0.0 +6968,,,0.12346100642106,0.1240783339262097,83274637.0,0.126419649889173,95000000.0,11418.61616706848,28600.53822851181,11418.61616706848,17158.9575817585,1.131810188293457,0.0 +7000,0.0053168698213994,0.122083380818367,,,,,,,,,,, +7182,,,0.1233746810954928,0.1239068020424281,83274637.0,0.1261435647168611,95000000.0,11774.297981977465,29179.98655867577,11774.297981977465,17382.02194261551,1.162308931350708,0.0 +7383,,,0.1221032197481472,0.1239043323143349,83274637.0,0.1262276595552143,95000000.0,12131.47572374344,29732.95296025276,12131.47572374344,17577.117379665375,1.191917896270752,0.0 +7500,0.0089366566389799,0.112597145140171,,,,,,,,,,, +7592,,,0.1207346899932191,0.1239396156478079,83274637.0,0.1263172912078857,95000000.0,12488.324034929276,30282.56272673607,12488.324034929276,17769.174802541733,1.2236275672912598,0.0 +7810,,,0.1229351218176018,0.1239536402518909,83274637.0,0.1263105004013864,95000000.0,12844.588040351868,30822.92870140076,12844.588040351868,17952.57149863243,1.2534687519073486,0.0 +8000,0.0061726788990199,0.118210382759571,,,,,,,,,,, +8002,,,0.1213618095305689,0.1238506327051705,83274637.0,0.126132372527273,95000000.0,13201.688807487488,31374.58929824829,13201.688807487488,18146.43815493584,1.283888578414917,0.0 +8207,,,,,,,,13558.222728013992,,,,,0.0 diff --git a/logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/trial_1/meta_data_0.json b/logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/trial_1/meta_data_0.json new file mode 100644 index 00000000..1be0eddb --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/criteo1tb_pytorch/trial_1/meta_data_0.json @@ -0,0 +1,72 @@ +{ + "workload.embed_dim": 128, + "workload.eval_batch_size": 8192, + "workload.eval_period_time_sec": 356, + "workload.max_allowed_runtime_sec": 8915, + "workload.num_dense_features": 13, + "workload.num_eval_train_examples": 83279872, + "workload.num_test_examples": 95000000, + "workload.num_train_examples": 4195197692, + "workload.num_validation_examples": 83274637, + "workload.step_hint": 10666, + "workload.target_metric_name": "loss", + "workload.test_target_value": 0.126041, + "workload.train_mean": 0.126041, + "workload.train_stddev": 0.126041, + "workload.use_layer_norm": false, + "workload.use_resnet": false, + "workload.validation_target_value": 0.123735, + "workload.vocab_size": 4194304, + "cpu.util.avg_percent_since_last": 5.9, + "cpu.freq.current": 2200.1959999999985, + "mem.total": 359053524992, + "mem.available": 349833490432, + "mem.used": 6059937792, + "mem.percent_used": 2.6, + "mem.read_bytes_since_boot": 36992892691968, + "mem.write_bytes_since_boot": 164841305600, + "net.bytes_sent_since_boot": 31340553, + "net.bytes_recv_since_boot": 31343646, + "gpu.count": 4, + "gpu.0.compute.util": 0.02, + "gpu.0.mem.util": 0.1278076171875, + "gpu.0.mem.total": 40960.0, + "gpu.0.mem.used": 5235.0, + "gpu.0.mem.free": 35092.0, + "gpu.0.temp.current": 35.0, + "gpu.1.compute.util": 0.0, + "gpu.1.mem.util": 0.1278076171875, + "gpu.1.mem.total": 40960.0, + "gpu.1.mem.used": 5235.0, + "gpu.1.mem.free": 35092.0, + "gpu.1.temp.current": 33.0, + "gpu.2.compute.util": 0.03, + "gpu.2.mem.util": 0.1278076171875, + "gpu.2.mem.total": 40960.0, + "gpu.2.mem.used": 5235.0, + "gpu.2.mem.free": 35092.0, + "gpu.2.temp.current": 33.0, + "gpu.3.compute.util": 0.0, + "gpu.3.mem.util": 0.1278076171875, + "gpu.3.mem.total": 40960.0, + "gpu.3.mem.used": 5235.0, + "gpu.3.mem.free": 35092.0, + "gpu.3.temp.current": 34.0, + "gpu.avg.compute.util": 0.0125, + "gpu.avg.mem.util": 0.1278076171875, + "gpu.avg.mem.total": 40960.0, + "gpu.avg.mem.used": 5235.0, + "gpu.avg.mem.free": 35092.0, + "gpu.avg.temp.current": 33.75, + "os_platform": "Linux-6.1.0-44-cloud-amd64-x86_64-with-glibc2.31", + "python_version": "3.11.10", + "python_compiler": "GCC 9.4.0", + "git_branch": "main", + "git_commit_hash": "b21be29be0a1573fb4f78f849aea019cdb520862", + "cpu_model_name": "Intel(R) Xeon(R) CPU @ 2.20GHz", + "cpu_count": 24, + "gpu_model_name": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_driver": "550.90.12", + "rng_seed": -826981189 +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_2/fastmri_pytorch/fastmri_pytorch_09-16-2026-07-15-03.log b/logs/self_tuning/ademamix_golden/study_2/fastmri_pytorch/fastmri_pytorch_09-16-2026-07-15-03.log new file mode 100644 index 00000000..90ad6159 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/fastmri_pytorch/fastmri_pytorch_09-16-2026-07-15-03.log @@ -0,0 +1,402 @@ +torchrun --redirects 1:0,2:0,3:0 --standalone --nnodes=1 --nproc_per_node=4 submission_runner.py --framework=pytorch --workload=fastmri --submission_path=submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py --data_dir=/data/fastmri --experiment_dir=/experiment_runs --experiment_name=submissions_a100/ademamix_golden/study_2 --overwrite=True --save_checkpoints=False --rng_seed=-482250772 --torch_compile=true --tuning_ruleset=self 2>&1 | tee -a /logs/fastmri_pytorch_09-16-2026-07-15-03.log +W0916 07:15:12.165000 9 site-packages/torch/distributed/run.py:803] +W0916 07:15:12.165000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0916 07:15:12.165000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0916 07:15:12.165000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-16 07:15:17.434716: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 07:15:17.434715: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 07:15:17.434715: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 07:15:17.434716: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789542917.459761 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789542917.459795 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789542917.459773 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789542917.459761 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789542917.467954 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789542917.467954 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789542917.467953 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789542917.467955 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789542917.495238 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789542917.495238 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789542917.495237 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789542917.495240 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789542917.495269 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789542917.495269 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789542917.495270 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789542917.495271 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789542917.495272 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789542917.495273 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789542917.495273 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789542917.495274 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789542917.495273 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789542917.495275 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789542917.495276 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789542917.495278 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789542942.563291 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789542942.563307 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789542942.563291 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789542942.614465 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W916 07:15:57.109509647 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0916 07:15:58.566359 140223838586048 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/fastmri_pytorch. +I0916 07:15:58.566357 140589535540416 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/fastmri_pytorch. +I0916 07:15:58.566357 140667564225728 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/fastmri_pytorch. +I0916 07:15:58.566387 139674797225152 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/fastmri_pytorch. +I0916 07:15:58.593064 140223838586048 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/fastmri_pytorch/trial_1. +I0916 07:15:58.911149 140223838586048 submission_runner.py:242] Initializing dataset. +I0916 07:15:58.911326 140223838586048 submission_runner.py:251] Initializing model. +I0916 07:15:59.136360 140223838586048 submission_runner.py:290] Performing `torch.compile`. +I0916 07:16:03.292139 140223838586048 submission_runner.py:294] Initializing optimizer. +I0916 07:16:03.292765 140223838586048 submission_runner.py:299] Initializing metrics bundle. +I0916 07:16:03.292914 140223838586048 submission_runner.py:321] Initializing checkpoint and logger. +I0916 07:16:03.295950 140223838586048 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_2/fastmri_pytorch/trial_1/meta_data_0.json. +I0916 07:16:03.296020 139674797225152 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 07:16:03.296024 140667564225728 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 07:16:03.296035 140589535540416 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 07:16:03.296142 140223838586048 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 07:16:03.296175 140667564225728 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 07:16:03.296179 139674797225152 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 07:16:03.296183 140589535540416 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 07:16:03.296198 140223838586048 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 07:16:03.691308 140223838586048 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_2/fastmri_pytorch/trial_1/flags_0.json. +I0916 07:16:03.747940 140223838586048 submission_runner.py:359] Starting training loop. +[rank2]:W0916 07:16:03.898000 40 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank1]:W0916 07:16:03.898000 39 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank3]:W0916 07:16:03.898000 41 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +WARNING:tensorflow:AutoGraph could not transform and will run it as-is. +Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. +Cause: (, (leaf_jax_array := getattr(leaf, '__jax_array__', None))) +To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert +W0916 07:16:05.370208 140223838586048 ag_logging.py:142] AutoGraph could not transform and will run it as-is. +Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. +Cause: (, (leaf_jax_array := getattr(leaf, '__jax_array__', None))) +To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert +WARNING:tensorflow:AutoGraph could not transform and will run it as-is. +Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. +Cause: (, (aval := get_aval(a))) +To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert +W0916 07:16:06.044820 140223838586048 ag_logging.py:142] AutoGraph could not transform and will run it as-is. +Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. +Cause: (, (aval := get_aval(a))) +To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert +[rank0]:W0916 07:18:46.604000 38 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +I0916 07:19:45.610238 140180543145728 logging_writer.py:48] [0] global_step=0, grad_norm=0.5, loss=0.897861 +I0916 07:19:45.789140 140223838586048 submission.py:307] 0) loss = 0.898, grad_norm = 0.500 +I0916 07:19:46.433442 140223838586048 spec.py:333] Evaluating on the training split. +[rank2]:W0916 07:20:20.916000 40 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] torch._dynamo hit config.recompile_limit (8) +[rank2]:W0916 07:20:20.916000 40 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] function: 'forward' (/algorithmic-efficiency/algoperf/workloads/fastmri/fastmri_pytorch/models.py:139) +[rank2]:W0916 07:20:20.916000 40 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] last reason: 5/7: GLOBAL_STATE changed: grad_mode +[rank2]:W0916 07:20:20.916000 40 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To log all recompilation reasons, use TORCH_LOGS="recompiles". +[rank2]:W0916 07:20:20.916000 40 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To diagnose recompilation issues, see https://pytorch.org/docs/main/torch.compiler_troubleshooting.html +[rank0]:W0916 07:20:20.917000 38 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] torch._dynamo hit config.recompile_limit (8) +[rank0]:W0916 07:20:20.917000 38 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] function: 'forward' (/algorithmic-efficiency/algoperf/workloads/fastmri/fastmri_pytorch/models.py:139) +[rank0]:W0916 07:20:20.917000 38 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] last reason: 5/7: GLOBAL_STATE changed: grad_mode +[rank0]:W0916 07:20:20.917000 38 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To log all recompilation reasons, use TORCH_LOGS="recompiles". +[rank0]:W0916 07:20:20.917000 38 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To diagnose recompilation issues, see https://pytorch.org/docs/main/torch.compiler_troubleshooting.html +[rank3]:W0916 07:20:20.931000 41 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] torch._dynamo hit config.recompile_limit (8) +[rank3]:W0916 07:20:20.931000 41 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] function: 'forward' (/algorithmic-efficiency/algoperf/workloads/fastmri/fastmri_pytorch/models.py:139) +[rank3]:W0916 07:20:20.931000 41 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] last reason: 5/7: GLOBAL_STATE changed: grad_mode +[rank3]:W0916 07:20:20.931000 41 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To log all recompilation reasons, use TORCH_LOGS="recompiles". +[rank3]:W0916 07:20:20.931000 41 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To diagnose recompilation issues, see https://pytorch.org/docs/main/torch.compiler_troubleshooting.html +[rank1]:W0916 07:20:20.983000 39 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] torch._dynamo hit config.recompile_limit (8) +[rank1]:W0916 07:20:20.983000 39 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] function: 'forward' (/algorithmic-efficiency/algoperf/workloads/fastmri/fastmri_pytorch/models.py:139) +[rank1]:W0916 07:20:20.983000 39 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] last reason: 5/7: GLOBAL_STATE changed: grad_mode +[rank1]:W0916 07:20:20.983000 39 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To log all recompilation reasons, use TORCH_LOGS="recompiles". +[rank1]:W0916 07:20:20.983000 39 site-packages/torch/_dynamo/convert_frame.py:1358] [5/8] To diagnose recompilation issues, see https://pytorch.org/docs/main/torch.compiler_troubleshooting.html +I0916 07:20:39.779754 140223838586048 spec.py:346] Evaluating on the validation split. +I0916 07:20:59.303418 140223838586048 spec.py:363] Evaluating on the test split. +I0916 07:21:17.975130 140223838586048 submission_runner.py:516] Time since start: 314.23s, Step: 1, {'train/ssim': 0.22977437291826522, 'train/loss': 0.8759488378252301, 'validation/ssim': 0.22483335375852911, 'validation/loss': 0.8778175779051772, 'validation/num_examples': 3554, 'test/ssim': 0.2473212707911547, 'test/loss': 0.8789597686793145, 'test/num_examples': 3581, 'score': 222.04259419441223, 'total_duration': 314.2272608280182, 'accumulated_submission_time': 222.04259419441223, 'accumulated_eval_time': 91.54177331924438, 'accumulated_logging_time': 0} +I0916 07:21:17.996080 140150268122880 logging_writer.py:48] [1] accumulated_eval_time=91.5418, accumulated_logging_time=0, accumulated_submission_time=222.043, global_step=1, preemption_count=0, score=222.043, test/loss=0.87896, test/num_examples=3581, test/ssim=0.247321, total_duration=314.227, train/loss=0.875949, train/ssim=0.229774, validation/loss=0.877818, validation/num_examples=3554, validation/ssim=0.224833 +I0916 07:21:18.745254 140150259730176 logging_writer.py:48] [1] global_step=1, grad_norm=0.5, loss=0.880655 +I0916 07:21:18.750798 140223838586048 submission.py:307] 1) loss = 0.881, grad_norm = 0.500 +I0916 07:21:18.853418 140150268122880 logging_writer.py:48] [2] global_step=2, grad_norm=0.5, loss=0.858001 +I0916 07:21:18.857663 140223838586048 submission.py:307] 2) loss = 0.858, grad_norm = 0.500 +I0916 07:21:18.945441 140150259730176 logging_writer.py:48] [3] global_step=3, grad_norm=0.5, loss=0.878988 +I0916 07:21:18.950929 140223838586048 submission.py:307] 3) loss = 0.879, grad_norm = 0.500 +I0916 07:21:19.045527 140150268122880 logging_writer.py:48] [4] global_step=4, grad_norm=0.5, loss=0.843815 +I0916 07:21:19.050755 140223838586048 submission.py:307] 4) loss = 0.844, grad_norm = 0.500 +I0916 07:21:19.148635 140150259730176 logging_writer.py:48] [5] global_step=5, grad_norm=0.5, loss=0.877881 +I0916 07:21:19.153304 140223838586048 submission.py:307] 5) loss = 0.878, grad_norm = 0.500 +I0916 07:21:19.248469 140150268122880 logging_writer.py:48] [6] global_step=6, grad_norm=0.5, loss=0.852087 +I0916 07:21:19.254560 140223838586048 submission.py:307] 6) loss = 0.852, grad_norm = 0.500 +I0916 07:21:19.359627 140150259730176 logging_writer.py:48] [7] global_step=7, grad_norm=0.5, loss=0.833812 +I0916 07:21:19.365941 140223838586048 submission.py:307] 7) loss = 0.834, grad_norm = 0.500 +I0916 07:21:19.448514 140150268122880 logging_writer.py:48] [8] global_step=8, grad_norm=0.5, loss=0.841128 +I0916 07:21:19.453469 140223838586048 submission.py:307] 8) loss = 0.841, grad_norm = 0.500 +I0916 07:21:19.540473 140150259730176 logging_writer.py:48] [9] global_step=9, grad_norm=0.5, loss=0.784977 +I0916 07:21:19.544710 140223838586048 submission.py:307] 9) loss = 0.785, grad_norm = 0.500 +I0916 07:21:19.642653 140150268122880 logging_writer.py:48] [10] global_step=10, grad_norm=0.5, loss=0.794825 +I0916 07:21:19.647905 140223838586048 submission.py:307] 10) loss = 0.795, grad_norm = 0.500 +I0916 07:21:19.742073 140150259730176 logging_writer.py:48] [11] global_step=11, grad_norm=0.5, loss=0.764428 +I0916 07:21:19.746437 140223838586048 submission.py:307] 11) loss = 0.764, grad_norm = 0.500 +I0916 07:21:19.844245 140150268122880 logging_writer.py:48] [12] global_step=12, grad_norm=0.5, loss=0.798895 +I0916 07:21:19.848038 140223838586048 submission.py:307] 12) loss = 0.799, grad_norm = 0.500 +I0916 07:21:19.943022 140150259730176 logging_writer.py:48] [13] global_step=13, grad_norm=0.5, loss=0.874529 +I0916 07:21:19.946905 140223838586048 submission.py:307] 13) loss = 0.875, grad_norm = 0.500 +I0916 07:21:20.050816 140150268122880 logging_writer.py:48] [14] global_step=14, grad_norm=0.5, loss=0.7575 +I0916 07:21:20.058222 140223838586048 submission.py:307] 14) loss = 0.758, grad_norm = 0.500 +I0916 07:21:20.151981 140150259730176 logging_writer.py:48] [15] global_step=15, grad_norm=0.5, loss=0.756859 +I0916 07:21:20.157058 140223838586048 submission.py:307] 15) loss = 0.757, grad_norm = 0.500 +I0916 07:21:20.246595 140150268122880 logging_writer.py:48] [16] global_step=16, grad_norm=0.5, loss=0.788762 +I0916 07:21:20.250840 140223838586048 submission.py:307] 16) loss = 0.789, grad_norm = 0.500 +I0916 07:21:20.341822 140150259730176 logging_writer.py:48] [17] global_step=17, grad_norm=0.5, loss=0.748746 +I0916 07:21:20.347151 140223838586048 submission.py:307] 17) loss = 0.749, grad_norm = 0.500 +I0916 07:21:20.445037 140150268122880 logging_writer.py:48] [18] global_step=18, grad_norm=0.5, loss=0.736565 +I0916 07:21:20.449274 140223838586048 submission.py:307] 18) loss = 0.737, grad_norm = 0.500 +I0916 07:21:20.539270 140150259730176 logging_writer.py:48] [19] global_step=19, grad_norm=0.5, loss=0.665562 +I0916 07:21:20.544351 140223838586048 submission.py:307] 19) loss = 0.666, grad_norm = 0.500 +I0916 07:21:20.642290 140150268122880 logging_writer.py:48] [20] global_step=20, grad_norm=0.5, loss=0.624444 +I0916 07:21:20.646170 140223838586048 submission.py:307] 20) loss = 0.624, grad_norm = 0.500 +I0916 07:21:20.734215 140150259730176 logging_writer.py:48] [21] global_step=21, grad_norm=0.5, loss=0.693257 +I0916 07:21:20.738137 140223838586048 submission.py:307] 21) loss = 0.693, grad_norm = 0.500 +I0916 07:21:20.829987 140150268122880 logging_writer.py:48] [22] global_step=22, grad_norm=0.5, loss=0.58791 +I0916 07:21:20.835769 140223838586048 submission.py:307] 22) loss = 0.588, grad_norm = 0.500 +I0916 07:21:20.927850 140150259730176 logging_writer.py:48] [23] global_step=23, grad_norm=0.5, loss=0.647663 +I0916 07:21:20.932074 140223838586048 submission.py:307] 23) loss = 0.648, grad_norm = 0.500 +I0916 07:21:21.028773 140150268122880 logging_writer.py:48] [24] global_step=24, grad_norm=0.5, loss=0.545512 +I0916 07:21:21.034892 140223838586048 submission.py:307] 24) loss = 0.546, grad_norm = 0.500 +I0916 07:21:21.127374 140150259730176 logging_writer.py:48] [25] global_step=25, grad_norm=0.5, loss=0.56284 +I0916 07:21:21.133261 140223838586048 submission.py:307] 25) loss = 0.563, grad_norm = 0.500 +I0916 07:21:21.231411 140150268122880 logging_writer.py:48] [26] global_step=26, grad_norm=0.5, loss=0.569839 +I0916 07:21:21.235829 140223838586048 submission.py:307] 26) loss = 0.570, grad_norm = 0.500 +I0916 07:21:21.324151 140150259730176 logging_writer.py:48] [27] global_step=27, grad_norm=0.5, loss=0.478572 +I0916 07:21:21.328399 140223838586048 submission.py:307] 27) loss = 0.479, grad_norm = 0.500 +I0916 07:21:21.427382 140150268122880 logging_writer.py:48] [28] global_step=28, grad_norm=0.5, loss=0.524543 +I0916 07:21:21.432628 140223838586048 submission.py:307] 28) loss = 0.525, grad_norm = 0.500 +I0916 07:21:21.527302 140150259730176 logging_writer.py:48] [29] global_step=29, grad_norm=0.5, loss=0.465479 +I0916 07:21:21.532274 140223838586048 submission.py:307] 29) loss = 0.465, grad_norm = 0.500 +I0916 07:21:21.623224 140150268122880 logging_writer.py:48] [30] global_step=30, grad_norm=0.499999, loss=0.461461 +I0916 07:21:21.627694 140223838586048 submission.py:307] 30) loss = 0.461, grad_norm = 0.500 +I0916 07:21:21.710118 140150259730176 logging_writer.py:48] [31] global_step=31, grad_norm=0.5, loss=0.511531 +I0916 07:21:21.715008 140223838586048 submission.py:307] 31) loss = 0.512, grad_norm = 0.500 +I0916 07:21:21.809506 140150268122880 logging_writer.py:48] [32] global_step=32, grad_norm=0.499999, loss=0.512397 +I0916 07:21:21.813955 140223838586048 submission.py:307] 32) loss = 0.512, grad_norm = 0.500 +I0916 07:21:21.901352 140150259730176 logging_writer.py:48] [33] global_step=33, grad_norm=0.499999, loss=0.491378 +I0916 07:21:21.906093 140223838586048 submission.py:307] 33) loss = 0.491, grad_norm = 0.500 +I0916 07:21:21.987554 140150268122880 logging_writer.py:48] [34] global_step=34, grad_norm=0.499999, loss=0.43253 +I0916 07:21:21.995080 140223838586048 submission.py:307] 34) loss = 0.433, grad_norm = 0.500 +I0916 07:21:22.083421 140150259730176 logging_writer.py:48] [35] global_step=35, grad_norm=0.499999, loss=0.494432 +I0916 07:21:22.088639 140223838586048 submission.py:307] 35) loss = 0.494, grad_norm = 0.500 +I0916 07:21:22.176404 140150268122880 logging_writer.py:48] [36] global_step=36, grad_norm=0.499999, loss=0.363611 +I0916 07:21:22.180828 140223838586048 submission.py:307] 36) loss = 0.364, grad_norm = 0.500 +I0916 07:21:22.264012 140150259730176 logging_writer.py:48] [37] global_step=37, grad_norm=0.499999, loss=0.498268 +I0916 07:21:22.268612 140223838586048 submission.py:307] 37) loss = 0.498, grad_norm = 0.500 +I0916 07:21:22.351923 140150268122880 logging_writer.py:48] [38] global_step=38, grad_norm=0.499999, loss=0.462526 +I0916 07:21:22.356097 140223838586048 submission.py:307] 38) loss = 0.463, grad_norm = 0.500 +I0916 07:21:22.433268 140150259730176 logging_writer.py:48] [39] global_step=39, grad_norm=0.499999, loss=0.406876 +I0916 07:21:22.437309 140223838586048 submission.py:307] 39) loss = 0.407, grad_norm = 0.500 +I0916 07:21:22.522870 140150268122880 logging_writer.py:48] [40] global_step=40, grad_norm=0.499999, loss=0.36276 +I0916 07:21:22.530928 140223838586048 submission.py:307] 40) loss = 0.363, grad_norm = 0.500 +I0916 07:21:22.607900 140150259730176 logging_writer.py:48] [41] global_step=41, grad_norm=0.499999, loss=0.482767 +I0916 07:21:22.611849 140223838586048 submission.py:307] 41) loss = 0.483, grad_norm = 0.500 +I0916 07:21:22.698766 140150268122880 logging_writer.py:48] [42] global_step=42, grad_norm=0.499999, loss=0.402957 +I0916 07:21:22.704281 140223838586048 submission.py:307] 42) loss = 0.403, grad_norm = 0.500 +I0916 07:21:22.780869 140150259730176 logging_writer.py:48] [43] global_step=43, grad_norm=0.472447, loss=0.413 +I0916 07:21:22.784811 140223838586048 submission.py:307] 43) loss = 0.413, grad_norm = 0.472 +I0916 07:21:22.865229 140150268122880 logging_writer.py:48] [44] global_step=44, grad_norm=0.481104, loss=0.395196 +I0916 07:21:22.869279 140223838586048 submission.py:307] 44) loss = 0.395, grad_norm = 0.481 +I0916 07:21:22.951483 140150259730176 logging_writer.py:48] [45] global_step=45, grad_norm=0.404969, loss=0.374793 +I0916 07:21:22.955638 140223838586048 submission.py:307] 45) loss = 0.375, grad_norm = 0.405 +I0916 07:21:23.036009 140150268122880 logging_writer.py:48] [46] global_step=46, grad_norm=0.491599, loss=0.314168 +I0916 07:21:23.041962 140223838586048 submission.py:307] 46) loss = 0.314, grad_norm = 0.492 +I0916 07:21:23.116938 140150259730176 logging_writer.py:48] [47] global_step=47, grad_norm=0.357276, loss=0.397125 +I0916 07:21:23.120852 140223838586048 submission.py:307] 47) loss = 0.397, grad_norm = 0.357 +I0916 07:21:23.204736 140150268122880 logging_writer.py:48] [48] global_step=48, grad_norm=0.366139, loss=0.45759 +I0916 07:21:23.208408 140223838586048 submission.py:307] 48) loss = 0.458, grad_norm = 0.366 +I0916 07:21:23.295027 140150259730176 logging_writer.py:48] [49] global_step=49, grad_norm=0.361767, loss=0.421283 +I0916 07:21:23.299934 140223838586048 submission.py:307] 49) loss = 0.421, grad_norm = 0.362 +I0916 07:21:23.469358 140150268122880 logging_writer.py:48] [50] global_step=50, grad_norm=0.423648, loss=0.478337 +I0916 07:21:23.473423 140223838586048 submission.py:307] 50) loss = 0.478, grad_norm = 0.424 +I0916 07:21:23.706522 140150259730176 logging_writer.py:48] [51] global_step=51, grad_norm=0.426705, loss=0.399767 +I0916 07:21:23.712629 140223838586048 submission.py:307] 51) loss = 0.400, grad_norm = 0.427 +I0916 07:21:23.988939 140150268122880 logging_writer.py:48] [52] global_step=52, grad_norm=0.324455, loss=0.344419 +I0916 07:21:23.992858 140223838586048 submission.py:307] 52) loss = 0.344, grad_norm = 0.324 +I0916 07:21:24.201606 140150259730176 logging_writer.py:48] [53] global_step=53, grad_norm=0.38798, loss=0.356642 +I0916 07:21:24.205764 140223838586048 submission.py:307] 53) loss = 0.357, grad_norm = 0.388 +I0916 07:21:24.362838 140150268122880 logging_writer.py:48] [54] global_step=54, grad_norm=0.286774, loss=0.314 +I0916 07:21:24.367011 140223838586048 submission.py:307] 54) loss = 0.314, grad_norm = 0.287 +I0916 07:21:24.566210 140150259730176 logging_writer.py:48] [55] global_step=55, grad_norm=0.276599, loss=0.367029 +I0916 07:21:24.571573 140223838586048 submission.py:307] 55) loss = 0.367, grad_norm = 0.277 +I0916 07:21:24.823296 140150268122880 logging_writer.py:48] [56] global_step=56, grad_norm=0.354364, loss=0.318959 +I0916 07:21:24.827493 140223838586048 submission.py:307] 56) loss = 0.319, grad_norm = 0.354 +I0916 07:21:25.094165 140150259730176 logging_writer.py:48] [57] global_step=57, grad_norm=0.323965, loss=0.297594 +I0916 07:21:25.098541 140223838586048 submission.py:307] 57) loss = 0.298, grad_norm = 0.324 +I0916 07:21:25.317451 140150268122880 logging_writer.py:48] [58] global_step=58, grad_norm=0.276836, loss=0.418412 +I0916 07:21:25.323546 140223838586048 submission.py:307] 58) loss = 0.418, grad_norm = 0.277 +I0916 07:21:25.480292 140150259730176 logging_writer.py:48] [59] global_step=59, grad_norm=0.265778, loss=0.343446 +I0916 07:21:25.484764 140223838586048 submission.py:307] 59) loss = 0.343, grad_norm = 0.266 +I0916 07:21:25.607199 140150268122880 logging_writer.py:48] [60] global_step=60, grad_norm=0.311008, loss=0.309403 +I0916 07:21:25.612123 140223838586048 submission.py:307] 60) loss = 0.309, grad_norm = 0.311 +I0916 07:21:25.699709 140150259730176 logging_writer.py:48] [61] global_step=61, grad_norm=0.290571, loss=0.425591 +I0916 07:21:25.704141 140223838586048 submission.py:307] 61) loss = 0.426, grad_norm = 0.291 +I0916 07:21:25.804972 140150268122880 logging_writer.py:48] [62] global_step=62, grad_norm=0.317048, loss=0.295668 +I0916 07:21:25.809549 140223838586048 submission.py:307] 62) loss = 0.296, grad_norm = 0.317 +I0916 07:21:25.878472 140150259730176 logging_writer.py:48] [63] global_step=63, grad_norm=0.215464, loss=0.333859 +I0916 07:21:25.883857 140223838586048 submission.py:307] 63) loss = 0.334, grad_norm = 0.215 +I0916 07:21:25.969407 140150268122880 logging_writer.py:48] [64] global_step=64, grad_norm=0.275132, loss=0.395989 +I0916 07:21:25.973866 140223838586048 submission.py:307] 64) loss = 0.396, grad_norm = 0.275 +I0916 07:21:26.053494 140150259730176 logging_writer.py:48] [65] global_step=65, grad_norm=0.302978, loss=0.324705 +I0916 07:21:26.058503 140223838586048 submission.py:307] 65) loss = 0.325, grad_norm = 0.303 +I0916 07:21:26.141900 140150268122880 logging_writer.py:48] [66] global_step=66, grad_norm=0.223141, loss=0.393613 +I0916 07:21:26.149168 140223838586048 submission.py:307] 66) loss = 0.394, grad_norm = 0.223 +I0916 07:21:26.247486 140150259730176 logging_writer.py:48] [67] global_step=67, grad_norm=0.298185, loss=0.34517 +I0916 07:21:26.252377 140223838586048 submission.py:307] 67) loss = 0.345, grad_norm = 0.298 +I0916 07:21:26.407672 140150268122880 logging_writer.py:48] [68] global_step=68, grad_norm=0.230756, loss=0.402445 +I0916 07:21:26.411651 140223838586048 submission.py:307] 68) loss = 0.402, grad_norm = 0.231 +I0916 07:21:26.536254 140150259730176 logging_writer.py:48] [69] global_step=69, grad_norm=0.498231, loss=0.286268 +I0916 07:21:26.540346 140223838586048 submission.py:307] 69) loss = 0.286, grad_norm = 0.498 +I0916 07:21:26.749079 140150268122880 logging_writer.py:48] [70] global_step=70, grad_norm=0.161763, loss=0.335496 +I0916 07:21:26.753146 140223838586048 submission.py:307] 70) loss = 0.335, grad_norm = 0.162 +I0916 07:21:26.888996 140150259730176 logging_writer.py:48] [71] global_step=71, grad_norm=0.286208, loss=0.3443 +I0916 07:21:26.894453 140223838586048 submission.py:307] 71) loss = 0.344, grad_norm = 0.286 +I0916 07:21:27.014705 140150268122880 logging_writer.py:48] [72] global_step=72, grad_norm=0.25674, loss=0.310381 +I0916 07:21:27.018673 140223838586048 submission.py:307] 72) loss = 0.310, grad_norm = 0.257 +I0916 07:21:27.205333 140150259730176 logging_writer.py:48] [73] global_step=73, grad_norm=0.175412, loss=0.253574 +I0916 07:21:27.209313 140223838586048 submission.py:307] 73) loss = 0.254, grad_norm = 0.175 +I0916 07:21:27.304620 140150268122880 logging_writer.py:48] [74] global_step=74, grad_norm=0.278534, loss=0.404277 +I0916 07:21:27.308998 140223838586048 submission.py:307] 74) loss = 0.404, grad_norm = 0.279 +I0916 07:21:27.477063 140150259730176 logging_writer.py:48] [75] global_step=75, grad_norm=0.216091, loss=0.314776 +I0916 07:21:27.481129 140223838586048 submission.py:307] 75) loss = 0.315, grad_norm = 0.216 +I0916 07:21:27.630525 140150268122880 logging_writer.py:48] [76] global_step=76, grad_norm=0.181521, loss=0.381124 +I0916 07:21:27.635561 140223838586048 submission.py:307] 76) loss = 0.381, grad_norm = 0.182 +I0916 07:21:27.737035 140150259730176 logging_writer.py:48] [77] global_step=77, grad_norm=0.257181, loss=0.313807 +I0916 07:21:27.742079 140223838586048 submission.py:307] 77) loss = 0.314, grad_norm = 0.257 +I0916 07:21:27.830416 140150268122880 logging_writer.py:48] [78] global_step=78, grad_norm=0.246648, loss=0.255278 +I0916 07:21:27.834507 140223838586048 submission.py:307] 78) loss = 0.255, grad_norm = 0.247 +I0916 07:21:28.006536 140150259730176 logging_writer.py:48] [79] global_step=79, grad_norm=0.191364, loss=0.296893 +I0916 07:21:28.011077 140223838586048 submission.py:307] 79) loss = 0.297, grad_norm = 0.191 +I0916 07:21:28.127674 140150268122880 logging_writer.py:48] [80] global_step=80, grad_norm=0.252281, loss=0.308268 +I0916 07:21:28.131922 140223838586048 submission.py:307] 80) loss = 0.308, grad_norm = 0.252 +I0916 07:21:28.255670 140150259730176 logging_writer.py:48] [81] global_step=81, grad_norm=0.211719, loss=0.342816 +I0916 07:21:28.259981 140223838586048 submission.py:307] 81) loss = 0.343, grad_norm = 0.212 +I0916 07:21:28.412352 140150268122880 logging_writer.py:48] [82] global_step=82, grad_norm=0.235485, loss=0.34385 +I0916 07:21:28.416430 140223838586048 submission.py:307] 82) loss = 0.344, grad_norm = 0.235 +I0916 07:21:28.662529 140150259730176 logging_writer.py:48] [83] global_step=83, grad_norm=0.429285, loss=0.326839 +I0916 07:21:28.668020 140223838586048 submission.py:307] 83) loss = 0.327, grad_norm = 0.429 +I0916 07:21:28.947439 140150268122880 logging_writer.py:48] [84] global_step=84, grad_norm=0.363259, loss=0.296739 +I0916 07:21:28.951326 140223838586048 submission.py:307] 84) loss = 0.297, grad_norm = 0.363 +I0916 07:21:29.297640 140150259730176 logging_writer.py:48] [85] global_step=85, grad_norm=0.260741, loss=0.431642 +I0916 07:21:29.302583 140223838586048 submission.py:307] 85) loss = 0.432, grad_norm = 0.261 +I0916 07:21:29.449392 140150268122880 logging_writer.py:48] [86] global_step=86, grad_norm=0.206987, loss=0.377121 +I0916 07:21:29.453202 140223838586048 submission.py:307] 86) loss = 0.377, grad_norm = 0.207 +I0916 07:21:29.608774 140150259730176 logging_writer.py:48] [87] global_step=87, grad_norm=0.303191, loss=0.400935 +I0916 07:21:29.612646 140223838586048 submission.py:307] 87) loss = 0.401, grad_norm = 0.303 +I0916 07:21:29.792600 140150268122880 logging_writer.py:48] [88] global_step=88, grad_norm=0.223758, loss=0.37476 +I0916 07:21:29.797982 140223838586048 submission.py:307] 88) loss = 0.375, grad_norm = 0.224 +I0916 07:21:30.020267 140150259730176 logging_writer.py:48] [89] global_step=89, grad_norm=0.235758, loss=0.42238 +I0916 07:21:30.024950 140223838586048 submission.py:307] 89) loss = 0.422, grad_norm = 0.236 +I0916 07:21:30.171244 140150268122880 logging_writer.py:48] [90] global_step=90, grad_norm=0.322482, loss=0.343125 +I0916 07:21:30.176597 140223838586048 submission.py:307] 90) loss = 0.343, grad_norm = 0.322 +I0916 07:21:30.312136 140150259730176 logging_writer.py:48] [91] global_step=91, grad_norm=0.19091, loss=0.309969 +I0916 07:21:30.317582 140223838586048 submission.py:307] 91) loss = 0.310, grad_norm = 0.191 +I0916 07:21:30.530430 140150268122880 logging_writer.py:48] [92] global_step=92, grad_norm=0.168817, loss=0.378857 +I0916 07:21:30.535257 140223838586048 submission.py:307] 92) loss = 0.379, grad_norm = 0.169 +I0916 07:21:30.641767 140150259730176 logging_writer.py:48] [93] global_step=93, grad_norm=0.24748, loss=0.392813 +I0916 07:21:30.645794 140223838586048 submission.py:307] 93) loss = 0.393, grad_norm = 0.247 +I0916 07:21:30.750418 140150268122880 logging_writer.py:48] [94] global_step=94, grad_norm=0.347045, loss=0.323211 +I0916 07:21:30.755015 140223838586048 submission.py:307] 94) loss = 0.323, grad_norm = 0.347 +I0916 07:21:30.832060 140150259730176 logging_writer.py:48] [95] global_step=95, grad_norm=0.181918, loss=0.445442 +I0916 07:21:30.837093 140223838586048 submission.py:307] 95) loss = 0.445, grad_norm = 0.182 +I0916 07:21:30.937141 140150268122880 logging_writer.py:48] [96] global_step=96, grad_norm=0.219249, loss=0.3928 +I0916 07:21:30.941207 140223838586048 submission.py:307] 96) loss = 0.393, grad_norm = 0.219 +I0916 07:21:31.140037 140150259730176 logging_writer.py:48] [97] global_step=97, grad_norm=0.169263, loss=0.433569 +I0916 07:21:31.144769 140223838586048 submission.py:307] 97) loss = 0.434, grad_norm = 0.169 +I0916 07:21:31.361238 140150268122880 logging_writer.py:48] [98] global_step=98, grad_norm=0.255392, loss=0.364885 +I0916 07:21:31.365509 140223838586048 submission.py:307] 98) loss = 0.365, grad_norm = 0.255 +I0916 07:21:31.524482 140150259730176 logging_writer.py:48] [99] global_step=99, grad_norm=0.164394, loss=0.349432 +I0916 07:21:31.530730 140223838586048 submission.py:307] 99) loss = 0.349, grad_norm = 0.164 +I0916 07:21:31.673236 140150268122880 logging_writer.py:48] [100] global_step=100, grad_norm=0.229611, loss=0.342303 +I0916 07:21:31.677741 140223838586048 submission.py:307] 100) loss = 0.342, grad_norm = 0.230 +I0916 07:22:43.332857 140150259730176 logging_writer.py:48] [500] global_step=500, grad_norm=0.211381, loss=0.322639 +I0916 07:22:43.337280 140223838586048 submission.py:307] 500) loss = 0.323, grad_norm = 0.211 +I0916 07:23:08.616597 140223838586048 spec.py:333] Evaluating on the training split. +I0916 07:23:10.782297 140223838586048 spec.py:346] Evaluating on the validation split. +I0916 07:23:13.507362 140223838586048 spec.py:363] Evaluating on the test split. +I0916 07:23:15.549226 140223838586048 submission_runner.py:516] Time since start: 431.80s, Step: 633, {'train/ssim': 0.7321959904261998, 'train/loss': 0.27623684065682547, 'validation/ssim': 0.7058234614606781, 'validation/loss': 0.3016399325583849, 'validation/num_examples': 3554, 'test/ssim': 0.7228558030883482, 'test/loss': 0.303718914169401, 'test/num_examples': 3581, 'score': 330.98101806640625, 'total_duration': 431.8015043735504, 'accumulated_submission_time': 330.98101806640625, 'accumulated_eval_time': 98.47485423088074, 'accumulated_logging_time': 0.030452728271484375} +I0916 07:23:15.570774 140150268122880 logging_writer.py:48] [633] accumulated_eval_time=98.4749, accumulated_logging_time=0.0304527, accumulated_submission_time=330.981, global_step=633, preemption_count=0, score=330.981, test/loss=0.303719, test/num_examples=3581, test/ssim=0.722856, total_duration=431.802, train/loss=0.276237, train/ssim=0.732196, validation/loss=0.30164, validation/num_examples=3554, validation/ssim=0.705823 +I0916 07:24:11.329889 140150259730176 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.104305, loss=0.220553 +I0916 07:24:11.332998 140223838586048 submission.py:307] 1000) loss = 0.221, grad_norm = 0.104 +I0916 07:24:35.470793 140150268122880 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.11624, loss=0.269453 +I0916 07:24:35.473955 140223838586048 submission.py:307] 1500) loss = 0.269, grad_norm = 0.116 +I0916 07:24:59.588059 140150259730176 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.147968, loss=0.227562 +I0916 07:24:59.591407 140223838586048 submission.py:307] 2000) loss = 0.228, grad_norm = 0.148 +I0916 07:25:06.115902 140223838586048 spec.py:333] Evaluating on the training split. +I0916 07:25:08.071158 140223838586048 spec.py:346] Evaluating on the validation split. +I0916 07:25:11.126994 140223838586048 spec.py:363] Evaluating on the test split. +I0916 07:25:13.691320 140223838586048 submission_runner.py:516] Time since start: 549.94s, Step: 2125, {'train/ssim': 0.7454218183244977, 'train/loss': 0.2647195202963693, 'validation/ssim': 0.7184533080024268, 'validation/loss': 0.29066576337841166, 'validation/num_examples': 3554, 'test/ssim': 0.7355945441217537, 'test/loss': 0.2922586218933259, 'test/num_examples': 3581, 'score': 439.7793185710907, 'total_duration': 549.943633556366, 'accumulated_submission_time': 439.7793185710907, 'accumulated_eval_time': 106.05038285255432, 'accumulated_logging_time': 0.06035017967224121} +I0916 07:25:13.713958 140150268122880 logging_writer.py:48] [2125] accumulated_eval_time=106.05, accumulated_logging_time=0.0603502, accumulated_submission_time=439.779, global_step=2125, preemption_count=0, score=439.779, test/loss=0.292259, test/num_examples=3581, test/ssim=0.735595, total_duration=549.944, train/loss=0.26472, train/ssim=0.745422, validation/loss=0.290666, validation/num_examples=3554, validation/ssim=0.718453 +I0916 07:25:32.448486 140150259730176 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.120698, loss=0.350598 +I0916 07:25:32.451632 140223838586048 submission.py:307] 2500) loss = 0.351, grad_norm = 0.121 +I0916 07:25:56.585175 140150268122880 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.0527325, loss=0.348939 +I0916 07:25:56.588337 140223838586048 submission.py:307] 3000) loss = 0.349, grad_norm = 0.053 +I0916 07:26:20.772470 140150259730176 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.123358, loss=0.277276 +I0916 07:26:20.775538 140223838586048 submission.py:307] 3500) loss = 0.277, grad_norm = 0.123 +I0916 07:26:44.927881 140150268122880 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.118749, loss=0.299882 +I0916 07:26:44.931054 140223838586048 submission.py:307] 4000) loss = 0.300, grad_norm = 0.119 +I0916 07:27:04.243079 140223838586048 spec.py:333] Evaluating on the training split. +I0916 07:27:06.206343 140223838586048 spec.py:346] Evaluating on the validation split. +I0916 07:27:08.947362 140223838586048 spec.py:363] Evaluating on the test split. +I0916 07:27:11.252356 140223838586048 submission_runner.py:516] Time since start: 667.50s, Step: 4391, {'train/ssim': 0.7482019151960101, 'train/loss': 0.2620269741330828, 'validation/ssim': 0.7205548133485158, 'validation/loss': 0.28866863948412, 'validation/num_examples': 3554, 'test/ssim': 0.7376641148596761, 'test/loss': 0.29015789442107653, 'test/num_examples': 3581, 'score': 548.4763679504395, 'total_duration': 667.5041129589081, 'accumulated_submission_time': 548.4763679504395, 'accumulated_eval_time': 113.05921244621277, 'accumulated_logging_time': 0.09099984169006348} +I0916 07:27:11.279864 140150259730176 logging_writer.py:48] [4391] accumulated_eval_time=113.059, accumulated_logging_time=0.0909998, accumulated_submission_time=548.476, global_step=4391, preemption_count=0, score=548.476, test/loss=0.290158, test/num_examples=3581, test/ssim=0.737664, total_duration=667.504, train/loss=0.262027, train/ssim=0.748202, validation/loss=0.288669, validation/num_examples=3554, validation/ssim=0.720555 +I0916 07:27:17.215324 140150268122880 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.0857709, loss=0.202923 +I0916 07:27:17.218395 140223838586048 submission.py:307] 4500) loss = 0.203, grad_norm = 0.086 +I0916 07:27:41.329311 140150259730176 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.15072, loss=0.23318 +I0916 07:27:41.332399 140223838586048 submission.py:307] 5000) loss = 0.233, grad_norm = 0.151 +I0916 07:28:05.430763 140150268122880 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.081373, loss=0.285547 +I0916 07:28:05.434010 140223838586048 submission.py:307] 5500) loss = 0.286, grad_norm = 0.081 +I0916 07:28:29.547939 140150259730176 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.0787743, loss=0.280878 +I0916 07:28:29.550962 140223838586048 submission.py:307] 6000) loss = 0.281, grad_norm = 0.079 +I0916 07:28:53.665653 140150268122880 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.142619, loss=0.244816 +I0916 07:28:53.668800 140223838586048 submission.py:307] 6500) loss = 0.245, grad_norm = 0.143 +I0916 07:29:01.843065 140223838586048 spec.py:333] Evaluating on the training split. +I0916 07:29:03.805552 140223838586048 spec.py:346] Evaluating on the validation split. +I0916 07:29:06.104768 140223838586048 spec.py:363] Evaluating on the test split. +I0916 07:29:08.138242 140223838586048 submission_runner.py:516] Time since start: 784.39s, Step: 6659, {'train/ssim': 0.7514567375183105, 'train/loss': 0.2595384120941162, 'validation/ssim': 0.7236262865125562, 'validation/loss': 0.2865283216863921, 'validation/num_examples': 3554, 'test/ssim': 0.7408052181871335, 'test/loss': 0.2879282107935109, 'test/num_examples': 3581, 'score': 657.1953451633453, 'total_duration': 784.3905413150787, 'accumulated_submission_time': 657.1953451633453, 'accumulated_eval_time': 119.35448980331421, 'accumulated_logging_time': 0.12905550003051758} +I0916 07:29:08.159135 140150259730176 logging_writer.py:48] [6659] accumulated_eval_time=119.354, accumulated_logging_time=0.129056, accumulated_submission_time=657.195, global_step=6659, preemption_count=0, score=657.195, test/loss=0.287928, test/num_examples=3581, test/ssim=0.740805, total_duration=784.391, train/loss=0.259538, train/ssim=0.751457, validation/loss=0.286528, validation/num_examples=3554, validation/ssim=0.723626 +I0916 07:29:25.252595 140150268122880 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.0735623, loss=0.283868 +I0916 07:29:25.255654 140223838586048 submission.py:307] 7000) loss = 0.284, grad_norm = 0.074 +I0916 07:29:49.368082 140150259730176 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.017678, loss=0.204691 +I0916 07:29:49.371182 140223838586048 submission.py:307] 7500) loss = 0.205, grad_norm = 0.018 +I0916 07:30:13.472319 140150268122880 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.0195369, loss=0.223605 +I0916 07:30:13.475504 140223838586048 submission.py:307] 8000) loss = 0.224, grad_norm = 0.020 +I0916 07:30:37.662932 140150259730176 logging_writer.py:48] [8500] global_step=8500, grad_norm=0.0524739, loss=0.325306 +I0916 07:30:37.665977 140223838586048 submission.py:307] 8500) loss = 0.325, grad_norm = 0.052 +I0916 07:30:58.684867 140223838586048 spec.py:333] Evaluating on the training split. +I0916 07:31:00.645631 140223838586048 spec.py:346] Evaluating on the validation split. +I0916 07:31:02.832321 140223838586048 spec.py:363] Evaluating on the test split. +I0916 07:31:04.834432 140223838586048 submission_runner.py:516] Time since start: 901.09s, Step: 8926, {'train/ssim': 0.7528419494628906, 'train/loss': 0.25860176767621723, 'validation/ssim': 0.7244977462674099, 'validation/loss': 0.28592638521296426, 'validation/num_examples': 3554, 'test/ssim': 0.7416641077693032, 'test/loss': 0.2873486409936994, 'test/num_examples': 3581, 'score': 765.8854074478149, 'total_duration': 901.0867321491241, 'accumulated_submission_time': 765.8854074478149, 'accumulated_eval_time': 125.50415134429932, 'accumulated_logging_time': 0.1583247184753418} +I0916 07:31:04.855881 140150268122880 logging_writer.py:48] [8926] accumulated_eval_time=125.504, accumulated_logging_time=0.158325, accumulated_submission_time=765.885, global_step=8926, preemption_count=0, score=765.885, test/loss=0.287349, test/num_examples=3581, test/ssim=0.741664, total_duration=901.087, train/loss=0.258602, train/ssim=0.752842, validation/loss=0.285926, validation/num_examples=3554, validation/ssim=0.724498 +I0916 07:31:05.425474 140150259730176 logging_writer.py:48] [8926] global_step=8926, preemption_count=0, score=765.885 +I0916 07:31:05.582615 140223838586048 submission_runner.py:857] Final fastmri score: 765.8854074478149 diff --git a/logs/self_tuning/ademamix_golden/study_2/fastmri_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_2/fastmri_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..f8023da4 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/fastmri_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,7 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/loss,test/num_examples,test/ssim,total_duration,train/loss,train/ssim,validation/loss,validation/num_examples,validation/ssim +91.54177331924438,0.0,222.04259419441223,1,0,222.04259419441223,0.8789597686793145,3581,0.2473212707911547,314.2272608280182,0.8759488378252301,0.2297743729182652,0.8778175779051772,3554,0.2248333537585291 +98.47485423088074,0.0304527282714843,330.98101806640625,633,0,330.98101806640625,0.303718914169401,3581,0.7228558030883482,431.8015043735504,0.2762368406568254,0.7321959904261998,0.3016399325583849,3554,0.7058234614606781 +106.05038285255432,0.0603501796722412,439.7793185710907,2125,0,439.7793185710907,0.2922586218933259,3581,0.7355945441217537,549.943633556366,0.2647195202963693,0.7454218183244977,0.2906657633784116,3554,0.7184533080024268 +113.05921244621275,0.0909998416900634,548.4763679504395,4391,0,548.4763679504395,0.2901578944210765,3581,0.7376641148596761,667.5041129589081,0.2620269741330828,0.7482019151960101,0.28866863948412,3554,0.7205548133485158 +119.3544898033142,0.1290555000305175,657.1953451633453,6659,0,657.1953451633453,0.2879282107935109,3581,0.7408052181871335,784.3905413150787,0.2595384120941162,0.7514567375183105,0.2865283216863921,3554,0.7236262865125562 +125.50415134429932,0.1583247184753418,765.8854074478149,8926,0,765.8854074478149,0.2873486409936994,3581,0.7416641077693032,901.0867321491241,0.25860176767621723,0.7528419494628906,0.28592638521296426,3554,0.7244977462674099 diff --git a/logs/self_tuning/ademamix_golden/study_2/fastmri_pytorch/trial_1/events.out.tfevents.1789542963.37feccc55827.38.0.v2 b/logs/self_tuning/ademamix_golden/study_2/fastmri_pytorch/trial_1/events.out.tfevents.1789542963.37feccc55827.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..c86b252a8c53e92b44a86b6b6ecd36a01d32550b GIT binary patch literal 30347 zcmbWrV# zNBRbZ>JFS55E^Y}=&U8yj1CBm3X7Z&5;jfOFDx=ZcS>|nh;G1-e;?&NEix!NAhJ)< z_QSmz(kc&JA1K=JAtLujnvRB8P24;rEGlY%xK*=u5}M8X6$V%jkr9i;Eu;K=LwqBn z+c;J7CHH6z}N5JtsuZG$l!@lLxAjVbvKbH zB$D}`Ym2*yWdcLO#`}hNM@0vO|BU$YA47!s@!7+S8zQ`nI*b*mKAH9K$N1%%>2w!& z6^o;PTob*hs30W#12?=O0K$ zFR_le%~aozAb;QJpfDWFakB|Hm<7)QG9c*d7iv)B5$!&~8PS0j`=t6ej7TE&+&)ph zRNT)iV=OilcW5-C&?%F>17dt9hld1^>6TFU8VHwq%si0?%+?TD5?DXs(E(A>aHX&I zk42p2=p%^`QPo}`qA_c>lrLUrk6MFK<_#?6E@@*$ww+0YuDF-DZT%<&L3!Q?+9BUs zMz0}i^0>|;Mdex~N>i*ZZV~n4run(_UW&bJ;d(4uv1E*CL-~&Bv7<%bdXe&;;vV8Z zqQj#9zkL0tMAlFH&zugOH)Bruk>sZX8**O1n0QEZ(wO9!iu;H=`TF@ynLH)LH#)%I zJ8H`K$wB{L&0(S5(Ls~xgX_>vbAWY2s8hgHUr~l72{jb!ixnD&{{P>HK%Ae#@d02} zzc}OVfQGo~A4B(w9uFjOCSoJ8QscOgu)x5e&_GE1Q45J~MuTRXG$j6gVcZMR;^8Eb ze)q@WV#r)MY=K!rNV_%VPenOyB&4^vr&umLG9X}b_>a5J+b?WNX!OrX;0;GM?4grx zL(+w7qrZr>?;ZSkxA1ntEs>l>Ir`f3_pq)Vib=KbUigi>7tRV&OoQvOqJl&;`FhA0 z{n;559WswG3Q7KcUYp{HS93?Zb>~1a`zomxR@8D8$D#sWfx`JNiJ&VaY+z{&c`2(0 z3e)Y`j6#a9@NL1Fi~hbo8$l7AMXH;q_+$I#jiAWCP9o?E8&p`ejLQbaKP}HQispQU zRdaWoix5A{7NBrYAk|G&wAri^4vGL1549^xxy|KnpxCEQs)ZH2 zlaBi^wXUG3wILC7#TisgPfvdW3PaWVjG`r9krC~RiVkKIazL?jB&ilw)N$`_`RD6j zgW|C-iJ&WrQE}yU&L&Wl+3K3snTj_v`*vCfqL7{k-R5wu}iU`*M#jD#Sg07H5MNhRm4`420 z`oCfnV!mS6^r5IYnx9tz3a_E0T3GRgyP{f3Kd}Ktnk$K*E7VajY3oFFQ21?p&nRU0 ziuvibs7R7*a|;xQQc1P2f_HZhby>e06yX^pg09d<#cY>Tvp`|c=Qc^9f;N2JIs`*Di;rX6K z&=ujR*jw3lJ}7oRYr`ny_zH(NgK;h#!`*^GVfl_!H&K!LZFwdrntvw|bVVF04$HPu z2F1~@3XGyHUvY20H7YvDoIVeV=4y&RtA!Q36?JoyxdbRAbw~tVk${S61zIma@qL{N zGmdtA#lmG4sBo1I&;>=a-K4sSimE?7%t4`&P9o@v<*0D9cb)@^xGkEDqCHw5Rp{uu^}S0_jWU9lb&1*t2;K{3O)2cwYZD-s^6p~5pu#|jjm zY)Ewz71ddr@V;;yNh0WqZK&AZb+I`pEEM$^g#usUFx>zZD^A{C3W~>Cq`HZU37IAR zVJ>X+NCaK62NkM^$8HCOoRtZqP~QrFUtpjhZbs)ZH2yE`jSqaP@8LP-Q& zaR?P}EAG^RVsETDqv*g_^u5s)6}rI(dxOF^fm91Cc=ttHnMH#^F?=0~pes(GLh@g)(1J zSKJmA@3mI10!7VNQZ1~g;m$>Iz;_WSltd$b4uG!6L&cHT7VAKv|GBLuzYgvWg zd8?OXzX65RYf>$&;H@YRpCUU@T&p4xbVUIw3`|sZf#UrXPe!4_S4?$~#<`ehuN(`C z@9Rjlu%e1P7sL1F+k;|eGKruoicz8Pp=u;3mK6Fjiq3q6#i(~U7pE0Z;~v%HHK`U> z{KHig4v2XGiu08ug03h-h0Dv@@u0Akn#d?r`HDolH>jw}9`On0B2}JL3oG7p6&`<| zz6*+MH4;HrRG?z-s7QNdNHv>ZDS6g>`; z2)d#M724@b+kj%?t0+dH&R0bCl0wDb$F1BzkZNH?Ial!|?B;n;tSloDbVYMiESZ;+ z2MRlC=`Q91YOYz6|!d4cR?{Uavr1T z!dK{b`-%$9Tr2DstCo{$VMQ5tF6{e1%Lm24EhK`jkVVBEn=JvLs0v)nD75&BUOh8W zG1_b+e#2Q8NveevycK1ekTV|?!E;CiT_J}G?X>peK=Dy(1*7Q7SA0|9JfJW`+eI5z zR8K`xEv$IXor}LC`{D^CLY+j=74oRiYk9*T6iz8?8AUg~!X}T?FWM{I$J1c-0aD#W zh5PFE>tQZZvq=P9p@fPvK?WM2IDdB&qv+08w06$J73J_u^)SrE$Wl@*ta!nli<+^? zeL!*KEs3BjR8Vp5`$GI~ux96WMxo7D?8-WciUEn+6hI-BN2;5sDE2AE@644SlL)#( z9Tnaqtp>ncT#(q!D0=V}j=if2RuMAwiAhQ@_i+d#Yhvbs;o|liN1v_-#i1xtEv!(QUWgH9AKtWr zx!ALaM9>wEsJM2a!VnbF0pA&gF<+56`VbDGZ}I)zpwNsU)xwI`s=F}aMWGz_i>0$j z1YI!_6|KiSI1h>&lC2-t|59wiSKQpd*Pw#ie*iXu?t&LGvo3imBbQIS9RDn192FqcHo6=PA6?W(jB z6xElF8HFidv2*wi9Kzx~p^HG_`h-*qD@?ABoTClKPue6wCW9t&>$B^Va8Xe znEj1Ih!_7Y1#^)xjZ_OOqH{Pas@S--5hzY3kO;bBA}Wq~ubBsmKOT573Uj`qPr*S{ zl(aTT21Tn@jL%5=Je=*EO zX(g!^R-C%98YAp_TjGcB79-rEf|WpFGK)me6@Q|_J59PDC=9y(#VBm}3Z>JW&)v_LmF)t>CR0)^tdJ>M zf{I!tZG4{gn=Of;E8Xl9EXTT`?OK7b;TmSC}`t3mL@#{#;}%xPn8N=AD}a3PVd$Ev%^f%O4|T(wu5w zMO|?x5p=~|R2;lE(G^xyf~O?Jp(xfucZm`Oj)$#j$;y_OInT>p3XK zE074fVm>Mqy0;DiMUufrW*l~WMc>w(?*_*Xy7LtjH>^mtuwuy25S$COhbL4(kusb_ z&=m=&=(Z`^0u(Jmk{QK7zT(mf&arykw}~@A(PIXw7FHN2aee~1yWMdGD4r}L5p=~O zR5&QP;^r!)xrwBl&+izwPEu>@kyIR&=pHi(Q=Y+ zEzHFr(Md*O&sQiraPEtqca<$bvAsR17FJwa$eGTJh=Nj3G&3L(bVVX6q`c0RfWlLJ zmQgtH6@AJ%=cow%myV!VtVF7X6_cap;9SH9-rfWXlkOyfu2_x=b*s<#_j(x?HyFiW zz9R5$JPzTcoAXRioOU49!U~hEMi`M_ea8e8ecVU{U9l1sg$KmJFc&dNcNoPGzGBK( z&K|XCQ7AqwUzA3wg%#3CiKv)4=>tAb>vx<)&=spuk>`+#-!_liQN$>Q@)h1X+i?h% zg_pj;To@c7)xwGogT`TmyR;lW^M5#-M9>v$QK7Lz%M9isVrm(qaO5ktYo5m;RC$at z28G^yQZ1}VDei|63EiU)!-{IPoJ7zS>rrvVz^Vuo+79m-#W22N_&&}zIfKt0J_Cxs zMv-b^h53HY-ECp9(-9PA(IkSd*ocZ#*}C|YSFLpwqZrOtWFMK07xLuWO?VnK9!jby ziWPri#QE;a>OgU342hsCl2DN(?}Oj`ht8KMs{d|q1YhAJ$vFg#d2|Gy(4M%KR0}J< zSK6XtRf~~NLGjma5SOQH8%vCMcw5DKiRZzM^|;&I#m#o;=QlL?WpcRy@{>K*fW72`@mQ zvYtfH6+2O3vf2RuF6nc;4x<>wSA>k-jYD|W)_pk4#k4J?T3FHl+Cz+Z{z>Em3j0GO zg09$wie}z_--Njc%(P$>E_{XBJI;3?-mSdwNz_G`Nwu(|i0g0e1 z_Mjr*YI7w}j4>L;C|vmpBMHtEcP+Y%34*!swk6fVis!TD;aqf!9sLRv$A*yzx?(RX zK4ni02ZiI4v5dlvuaHwbi$fSXFG&IvTgynbuwq>0D2!O6;fl|JT&yM$bVV8}%C6R~ z0!2aVKt?f|uW+`%fkUtl?|2myNlK(zSW#@w`3~g3NmqQH_Mh$~g047#iaz0;vFojR z63Hms`HCgQoF^!(Q&%kjh0iNeEvz`M9>BQ}Za3P&iqii`BIt@ks3_cRq6l+g8W_(g z#_$!I0}5~mtsdmNfue8aXfK&@B9>;RdQRhC*P6ovkYZ5_M97Dy$OC>rm7sLNZW)x%jiglkkdsI@s z1orO9ok+E?Vn$IfT$DS43UffQrYDJ@D^8%o{YtOrpqQzf$|%P16`87>Z#etvw%!19 zVQ5LJg%!htV^Q(7#}ao?Y#2f!=!#5Kq^+_a0t&M(hZuzyUr{rYv#9NBQtd!7xHqX5 zR@^?Aj*5NH-{Ifdms^qux*{7D6+6e{$y{#iQAXj-S4gxi#UaF)-1G!R^mI}!thnW1 zh7p5vcFMq9M9m=)bj2xDNJ-x70dt|Okj*H3_zJnsoE3HNs^KY6?Va0{nKB(~c zwg>0pQg0GLSDZnG!_8_*Q23}_VHCc6MQ$DE997=tBmUK)We-vEaWry)Xp4h()=VD+MsTNkeb<@TOnbdPHLE(RuM9>u% zQ4zIg1^zl|*zYx?@aHSs2XK0~ht95+Fc&4Zq*_?fSB>*p#PkZ6JWwohB@uMRWmMRh zZaf0-*y|a3ucH1tkV=7e&5IVm$6uExXU(wch6=jOCGO&Ug+<6ny@w|HF%AzRY~LmIyUu{@OX zOo+C;K^naD&LD#g`PI879~?5q;bi+{OoyZ;S@fjFV_F)%5m)?ywQYvO`vEJ@u}S~w zZId5M)jz?^_qoa@*3Ufeb(8X~`=RfpL2U?R7~~6OG?vN6b3PNz|8skyMZ@a3e0+zq z$g64QlE;Cwas3e=}jOO+%`7hsrggQ&nv0zrS=+{Uc`|f3P~W z!nPqVeO}jXqL+1SUen$>8S~i!UfX4w_0%K_^q>rz z@_X-{tWxfh2>YXBpL^0Pe!Rc7OyH$%3krPkzI*fKT^y|NSZlUicz=Aq`JN?cjOQ=$ zf@>0H&t_1siNp-NCUw?JP6K3zd68^AYLuA^#Lx@V9 z<0XKo4P!H?khm;_d|I`>2SDsjJ#SYJ$rE^O(wi1R2ubf~;|-7tm)UY*$cwohFhir~ zLmz+y=Cc`8h{P9!?BBXF6`IYctCJP$Avfe`9jp3|6>gyqGbQi~-;&#cA&vJ?r-cj9 z;})Cv8T!oE2W-f%I#%MFzBrt3XPhJq zsvlwIwJO*|;g0oo1m_h=pBA*o+d#2eD5J6LP4XJN+66-bI3#f!Aj(jz5B{bJ&YkL*Op)xX+dg?~mK{ ztMQt|B%GNF*JSDwHiLRiY*g@?7)!@v>wUSjt42NKguwfbI_}|oPIOV-^fy4RuVu@H zA$4CzBSi11-W`BQZ)G#6kT`RM=sRk8z>F+dXwbbLk|ywi8qI3X=ZTu-vWWnRU&)pW zL!KwsA;f>J#uOArdnX5@m=>-ilTZ;|(RqJ(_YKpD*L-mJukTkb9J*n}Se6Fv;F-@${(u3Ap zXTm1^r`GGEc*`DUeuOofSU>Z;4~vQSIjz_KwoMLXMA-{vG?uMe#yM0(wv8MNt=D%H zoAH~icWIx2@z8qB#6p3kZ9|~-)*WpaW9hnf(0Zfd+0=h;y-uo6vGqQ9s^J4-?M)`q)y-s6lO-82YPR% z)Xe}$hhuEHFr;YLE}Ri3x6{D@xp<1rphAlGA*9(}{|x~7*4@v(9#SFj$^e5~%kXM; zH)`<@Kq?H`a$!h>%XWke5|``+$a^z3g9?euLP)nPg?xZ`XG|Ja4=EM+8}FX!GZ0eU zW}qoRTyxlRVMva+9cFBtQF#R*ju+SrDn$JnLOQfroepP+sv%;hddM7O+8H9o+XkDq z^N2bF2$66R3~AieOrOq0&$AHe577BTJlT+6ogr3PbH0!@me0nI@809-NsY&J{|M*K z@qeoB1!stMlh~yHbcUEZYMTgVULk@_te<(_86qgf5-;~2tykudu{K61qp{2-m~%Qh zmR@2AXNXI)*o@yiLo7aUYY?0vQs=W-{4+$nqZYRQrS0D08RGU*w*D7qi0?{v7_=g) zz!%OCPgk=+O+7>KhNrV&t0bHu&TnQ@|NRURck~^eA@UV2Pk}SU(Vc8w)6Njxo8!Hc zu=j2>>;;KwY$SCrP|Cx-;FLtdGB`t6Ww0r~cZSHl*0&1w$8WYHUF)}%0)fBv*BQNn z<7xfa5?1&vN48vef4skYEpGH-7pHE3YjW0w&7fYBGlh6fI@;%H1EglpXwQ1cb%8(5 zj^5~mkbpZQTL8pAoh=uJB;+s0UlZKEzE}g0gp+Ir72;ElkP~|i+5+U6tCw#*y z({8EZ{K;S9;qxN^67I#83quNJwjrb>A>cSb4o_e+s1S)S2w9t4`w}3{=1&T&hhzx+ z`Qyq&&L44ZuG4o0$n8Y7To_XP&JCB=?Xo^A0phog&7eZ^nh%0sL`+1UW1(waEbbFp z4=EZ*YrQs^oYwo-%(M7sPCH%+hBR&lQGuM+D{-+)B(&Z~6>P|_TJOOaPU~HesbvmH zU25n_jmPxCpVNA$i0xZI>z)0bP5Mu*H+Qw^Vwm}G>AvK8*Uvn!^&WZ5X}y0s=E_3G zhSowEjb&riIIZ`D<AJm1d z|3&LvI?xw`!Z)T)h1UD32OHGX*2^27%+8oa(0WY`+0=h;z3P&+*m}LM*3N>~JG390 z*QC~4tfq-Oyk>!yH0%Y}ZP-ZaUJxULd%=Y@Csd*JMmex4zt?&<&1 | tee -a /logs/finewebedu_lm_pytorch_09-11-2026-22-36-05.log +W0911 22:36:07.130000 9 site-packages/torch/distributed/run.py:803] +W0911 22:36:07.130000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0911 22:36:07.130000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0911 22:36:07.130000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-11 22:36:08.515389: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-11 22:36:08.515392: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-11 22:36:08.515389: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-11 22:36:08.515387: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789166168.536357 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789166168.536359 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789166168.536358 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789166168.536355 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789166168.543270 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789166168.543278 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789166168.543278 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789166168.543284 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789166168.560451 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789166168.560452 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789166168.560451 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789166168.560451 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789166168.560474 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789166168.560475 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789166168.560476 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789166168.560477 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789166168.560477 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789166168.560478 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789166168.560479 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789166168.560479 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789166168.560480 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789166168.560481 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789166168.560483 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789166168.560485 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789166174.340835 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789166174.431148 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789166174.559634 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789166174.560596 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W911 22:36:16.587631657 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0911 22:36:17.029990 139924186363072 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/finewebedu_lm_pytorch. +I0911 22:36:17.029990 140532213929152 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/finewebedu_lm_pytorch. +I0911 22:36:17.029990 140564353668288 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/finewebedu_lm_pytorch. +I0911 22:36:17.030014 139820898444480 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/finewebedu_lm_pytorch. +I0911 22:36:17.053188 139924186363072 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/finewebedu_lm_pytorch/trial_1. +I0911 22:36:17.306696 139924186363072 submission_runner.py:242] Initializing dataset. +I0911 22:36:17.306861 139924186363072 submission_runner.py:251] Initializing model. +I0911 22:36:23.742303 139924186363072 submission_runner.py:290] Performing `torch.compile`. +I0911 22:36:25.135978 140532213929152 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0911 22:36:25.136133 140532213929152 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0911 22:36:25.136536 139820898444480 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0911 22:36:25.136683 139820898444480 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0911 22:36:25.136739 139924186363072 submission_runner.py:294] Initializing optimizer. +I0911 22:36:25.137432 139924186363072 submission_runner.py:299] Initializing metrics bundle. +I0911 22:36:25.137591 139924186363072 submission_runner.py:321] Initializing checkpoint and logger. +I0911 22:36:25.138004 139924186363072 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_2/finewebedu_lm_pytorch/trial_1/meta_data_0.json. +I0911 22:36:25.138181 139924186363072 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0911 22:36:25.138237 139924186363072 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0911 22:36:25.138602 140564353668288 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0911 22:36:25.138750 140564353668288 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0911 22:36:25.362057 139924186363072 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_2/finewebedu_lm_pytorch/trial_1/flags_0.json. +I0911 22:36:25.403531 139924186363072 submission_runner.py:359] Starting training loop. +[rank0]:W0911 22:36:27.264000 38 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank1]:W0911 22:36:27.303000 39 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank2]:W0911 22:36:27.328000 40 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank3]:W0911 22:36:27.381000 41 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +I0911 22:37:08.915529 139897931024128 logging_writer.py:48] [0] global_step=0, grad_norm=0.5, loss=11.0496 +I0911 22:37:08.946465 139924186363072 submission.py:307] 0) loss = 11.050, grad_norm = 0.500 +I0911 22:37:09.552608 139924186363072 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +I0911 22:38:04.319061 139924186363072 spec.py:346] Evaluating on the validation split. +I0911 22:40:55.362933 139924186363072 spec.py:363] Evaluating on the test split. +I0911 22:40:55.363616 139924186363072 submission_runner.py:516] Time since start: 269.96s, Step: 1, {'train/loss': 11.046409606933594, 'train/ppl': 62718.36628502924, 'validation/loss': 11.04719469309463, 'validation/ppl': 62767.6249400565, 'validation/num_examples': 100000, 'test/loss': 0.0, 'test/ppl': 1.0, 'test/num_examples': 0, 'score': 43.54396414756775, 'total_duration': 269.9600627422333, 'accumulated_submission_time': 43.54396414756775, 'accumulated_eval_time': 225.8109667301178, 'accumulated_logging_time': 0} +I0911 22:40:55.382629 139895989171968 logging_writer.py:48] [1] accumulated_eval_time=225.811, accumulated_logging_time=0, accumulated_submission_time=43.544, global_step=1, preemption_count=0, score=43.544, test/loss=0, test/num_examples=0, test/ppl=1, total_duration=269.96, train/loss=11.0464, train/ppl=62718.4, validation/loss=11.0472, validation/num_examples=100000, validation/ppl=62767.6 +I0911 22:40:57.076719 139895980779264 logging_writer.py:48] [1] global_step=1, grad_norm=0.5, loss=11.0473 +I0911 22:40:57.079942 139924186363072 submission.py:307] 1) loss = 11.047, grad_norm = 0.500 +I0911 22:40:57.519345 139895989171968 logging_writer.py:48] [2] global_step=2, grad_norm=0.5, loss=11.0255 +I0911 22:40:57.522397 139924186363072 submission.py:307] 2) loss = 11.026, grad_norm = 0.500 +I0911 22:40:57.952457 139895980779264 logging_writer.py:48] [3] global_step=3, grad_norm=0.5, loss=10.9925 +I0911 22:40:57.955454 139924186363072 submission.py:307] 3) loss = 10.992, grad_norm = 0.500 +I0911 22:40:58.383925 139895989171968 logging_writer.py:48] [4] global_step=4, grad_norm=0.5, loss=10.9373 +I0911 22:40:58.386864 139924186363072 submission.py:307] 4) loss = 10.937, grad_norm = 0.500 +I0911 22:40:58.816804 139895980779264 logging_writer.py:48] [5] global_step=5, grad_norm=0.5, loss=10.8742 +I0911 22:40:58.819698 139924186363072 submission.py:307] 5) loss = 10.874, grad_norm = 0.500 +I0911 22:40:59.247909 139895989171968 logging_writer.py:48] [6] global_step=6, grad_norm=0.5, loss=10.7828 +I0911 22:40:59.250871 139924186363072 submission.py:307] 6) loss = 10.783, grad_norm = 0.500 +I0911 22:40:59.681566 139895980779264 logging_writer.py:48] [7] global_step=7, grad_norm=0.5, loss=10.682 +I0911 22:40:59.684458 139924186363072 submission.py:307] 7) loss = 10.682, grad_norm = 0.500 +I0911 22:41:00.114787 139895989171968 logging_writer.py:48] [8] global_step=8, grad_norm=0.5, loss=10.5921 +I0911 22:41:00.117657 139924186363072 submission.py:307] 8) loss = 10.592, grad_norm = 0.500 +I0911 22:41:00.547101 139895980779264 logging_writer.py:48] [9] global_step=9, grad_norm=0.5, loss=10.4631 +I0911 22:41:00.550055 139924186363072 submission.py:307] 9) loss = 10.463, grad_norm = 0.500 +I0911 22:41:00.979000 139895989171968 logging_writer.py:48] [10] global_step=10, grad_norm=0.5, loss=10.3423 +I0911 22:41:00.981962 139924186363072 submission.py:307] 10) loss = 10.342, grad_norm = 0.500 +I0911 22:41:01.411929 139895980779264 logging_writer.py:48] [11] global_step=11, grad_norm=0.5, loss=10.1949 +I0911 22:41:01.414814 139924186363072 submission.py:307] 11) loss = 10.195, grad_norm = 0.500 +I0911 22:41:01.845218 139895989171968 logging_writer.py:48] [12] global_step=12, grad_norm=0.5, loss=10.0545 +I0911 22:41:01.848082 139924186363072 submission.py:307] 12) loss = 10.055, grad_norm = 0.500 +I0911 22:41:02.275828 139895980779264 logging_writer.py:48] [13] global_step=13, grad_norm=0.5, loss=9.93079 +I0911 22:41:02.278743 139924186363072 submission.py:307] 13) loss = 9.931, grad_norm = 0.500 +I0911 22:41:02.709138 139895989171968 logging_writer.py:48] [14] global_step=14, grad_norm=0.5, loss=9.80953 +I0911 22:41:02.712004 139924186363072 submission.py:307] 14) loss = 9.810, grad_norm = 0.500 +I0911 22:41:03.142596 139895980779264 logging_writer.py:48] [15] global_step=15, grad_norm=0.5, loss=9.72678 +I0911 22:41:03.145537 139924186363072 submission.py:307] 15) loss = 9.727, grad_norm = 0.500 +I0911 22:41:03.575879 139895989171968 logging_writer.py:48] [16] global_step=16, grad_norm=0.5, loss=9.64841 +I0911 22:41:03.578754 139924186363072 submission.py:307] 16) loss = 9.648, grad_norm = 0.500 +I0911 22:41:04.008380 139895980779264 logging_writer.py:48] [17] global_step=17, grad_norm=0.5, loss=9.54471 +I0911 22:41:04.011335 139924186363072 submission.py:307] 17) loss = 9.545, grad_norm = 0.500 +I0911 22:41:04.440063 139895989171968 logging_writer.py:48] [18] global_step=18, grad_norm=0.5, loss=9.48422 +I0911 22:41:04.443009 139924186363072 submission.py:307] 18) loss = 9.484, grad_norm = 0.500 +I0911 22:41:04.872679 139895980779264 logging_writer.py:48] [19] global_step=19, grad_norm=0.5, loss=9.4516 +I0911 22:41:04.875602 139924186363072 submission.py:307] 19) loss = 9.452, grad_norm = 0.500 +I0911 22:41:05.301217 139895989171968 logging_writer.py:48] [20] global_step=20, grad_norm=0.5, loss=9.36653 +I0911 22:41:05.304265 139924186363072 submission.py:307] 20) loss = 9.367, grad_norm = 0.500 +I0911 22:41:05.733054 139895980779264 logging_writer.py:48] [21] global_step=21, grad_norm=0.5, loss=9.37139 +I0911 22:41:05.736059 139924186363072 submission.py:307] 21) loss = 9.371, grad_norm = 0.500 +I0911 22:41:06.167038 139895989171968 logging_writer.py:48] [22] global_step=22, grad_norm=0.5, loss=9.32098 +I0911 22:41:06.169915 139924186363072 submission.py:307] 22) loss = 9.321, grad_norm = 0.500 +I0911 22:41:06.600763 139895980779264 logging_writer.py:48] [23] global_step=23, grad_norm=0.5, loss=9.29033 +I0911 22:41:06.603759 139924186363072 submission.py:307] 23) loss = 9.290, grad_norm = 0.500 +I0911 22:41:07.032448 139895989171968 logging_writer.py:48] [24] global_step=24, grad_norm=0.5, loss=9.2549 +I0911 22:41:07.035445 139924186363072 submission.py:307] 24) loss = 9.255, grad_norm = 0.500 +I0911 22:41:07.464476 139895980779264 logging_writer.py:48] [25] global_step=25, grad_norm=0.5, loss=9.19033 +I0911 22:41:07.467503 139924186363072 submission.py:307] 25) loss = 9.190, grad_norm = 0.500 +I0911 22:41:07.896329 139895989171968 logging_writer.py:48] [26] global_step=26, grad_norm=0.5, loss=9.23667 +I0911 22:41:07.899200 139924186363072 submission.py:307] 26) loss = 9.237, grad_norm = 0.500 +I0911 22:41:08.328846 139895980779264 logging_writer.py:48] [27] global_step=27, grad_norm=0.5, loss=9.2246 +I0911 22:41:08.331815 139924186363072 submission.py:307] 27) loss = 9.225, grad_norm = 0.500 +I0911 22:41:08.760985 139895989171968 logging_writer.py:48] [28] global_step=28, grad_norm=0.5, loss=9.17793 +I0911 22:41:08.763969 139924186363072 submission.py:307] 28) loss = 9.178, grad_norm = 0.500 +I0911 22:41:09.191697 139895980779264 logging_writer.py:48] [29] global_step=29, grad_norm=0.5, loss=9.1595 +I0911 22:41:09.194584 139924186363072 submission.py:307] 29) loss = 9.159, grad_norm = 0.500 +I0911 22:41:09.623240 139895989171968 logging_writer.py:48] [30] global_step=30, grad_norm=0.5, loss=9.12984 +I0911 22:41:09.626258 139924186363072 submission.py:307] 30) loss = 9.130, grad_norm = 0.500 +I0911 22:41:10.056887 139895980779264 logging_writer.py:48] [31] global_step=31, grad_norm=0.5, loss=9.13117 +I0911 22:41:10.059791 139924186363072 submission.py:307] 31) loss = 9.131, grad_norm = 0.500 +I0911 22:41:10.489024 139895989171968 logging_writer.py:48] [32] global_step=32, grad_norm=0.5, loss=9.01496 +I0911 22:41:10.491866 139924186363072 submission.py:307] 32) loss = 9.015, grad_norm = 0.500 +I0911 22:41:10.921095 139895980779264 logging_writer.py:48] [33] global_step=33, grad_norm=0.5, loss=9.06578 +I0911 22:41:10.924001 139924186363072 submission.py:307] 33) loss = 9.066, grad_norm = 0.500 +I0911 22:41:11.353360 139895989171968 logging_writer.py:48] [34] global_step=34, grad_norm=0.5, loss=9.02351 +I0911 22:41:11.356280 139924186363072 submission.py:307] 34) loss = 9.024, grad_norm = 0.500 +I0911 22:41:11.784028 139895980779264 logging_writer.py:48] [35] global_step=35, grad_norm=0.5, loss=8.96244 +I0911 22:41:11.787259 139924186363072 submission.py:307] 35) loss = 8.962, grad_norm = 0.500 +I0911 22:41:12.215432 139895989171968 logging_writer.py:48] [36] global_step=36, grad_norm=0.5, loss=8.96373 +I0911 22:41:12.218367 139924186363072 submission.py:307] 36) loss = 8.964, grad_norm = 0.500 +I0911 22:41:12.648976 139895980779264 logging_writer.py:48] [37] global_step=37, grad_norm=0.5, loss=8.9185 +I0911 22:41:12.651935 139924186363072 submission.py:307] 37) loss = 8.919, grad_norm = 0.500 +I0911 22:41:13.081842 139895989171968 logging_writer.py:48] [38] global_step=38, grad_norm=0.5, loss=8.90638 +I0911 22:41:13.084802 139924186363072 submission.py:307] 38) loss = 8.906, grad_norm = 0.500 +I0911 22:41:13.512857 139895980779264 logging_writer.py:48] [39] global_step=39, grad_norm=0.5, loss=8.83714 +I0911 22:41:13.515810 139924186363072 submission.py:307] 39) loss = 8.837, grad_norm = 0.500 +I0911 22:41:13.946045 139895989171968 logging_writer.py:48] [40] global_step=40, grad_norm=0.5, loss=8.82403 +I0911 22:41:13.948968 139924186363072 submission.py:307] 40) loss = 8.824, grad_norm = 0.500 +I0911 22:41:14.378341 139895980779264 logging_writer.py:48] [41] global_step=41, grad_norm=0.5, loss=8.86231 +I0911 22:41:14.381207 139924186363072 submission.py:307] 41) loss = 8.862, grad_norm = 0.500 +I0911 22:41:14.810972 139895989171968 logging_writer.py:48] [42] global_step=42, grad_norm=0.5, loss=8.81331 +I0911 22:41:14.813920 139924186363072 submission.py:307] 42) loss = 8.813, grad_norm = 0.500 +I0911 22:41:15.243034 139895980779264 logging_writer.py:48] [43] global_step=43, grad_norm=0.5, loss=8.74036 +I0911 22:41:15.245940 139924186363072 submission.py:307] 43) loss = 8.740, grad_norm = 0.500 +I0911 22:41:15.673768 139895989171968 logging_writer.py:48] [44] global_step=44, grad_norm=0.5, loss=8.72373 +I0911 22:41:15.676675 139924186363072 submission.py:307] 44) loss = 8.724, grad_norm = 0.500 +I0911 22:41:16.105479 139895980779264 logging_writer.py:48] [45] global_step=45, grad_norm=0.5, loss=8.73507 +I0911 22:41:16.108373 139924186363072 submission.py:307] 45) loss = 8.735, grad_norm = 0.500 +I0911 22:41:16.538310 139895989171968 logging_writer.py:48] [46] global_step=46, grad_norm=0.5, loss=8.70523 +I0911 22:41:16.541520 139924186363072 submission.py:307] 46) loss = 8.705, grad_norm = 0.500 +I0911 22:41:16.969838 139895980779264 logging_writer.py:48] [47] global_step=47, grad_norm=0.5, loss=8.64766 +I0911 22:41:16.972767 139924186363072 submission.py:307] 47) loss = 8.648, grad_norm = 0.500 +I0911 22:41:17.401986 139895989171968 logging_writer.py:48] [48] global_step=48, grad_norm=0.5, loss=8.7097 +I0911 22:41:17.404895 139924186363072 submission.py:307] 48) loss = 8.710, grad_norm = 0.500 +I0911 22:41:17.833490 139895980779264 logging_writer.py:48] [49] global_step=49, grad_norm=0.5, loss=8.6452 +I0911 22:41:17.836323 139924186363072 submission.py:307] 49) loss = 8.645, grad_norm = 0.500 +I0911 22:41:18.264454 139895989171968 logging_writer.py:48] [50] global_step=50, grad_norm=0.5, loss=8.64683 +I0911 22:41:18.267387 139924186363072 submission.py:307] 50) loss = 8.647, grad_norm = 0.500 +I0911 22:41:18.693922 139895980779264 logging_writer.py:48] [51] global_step=51, grad_norm=0.5, loss=8.58931 +I0911 22:41:18.696809 139924186363072 submission.py:307] 51) loss = 8.589, grad_norm = 0.500 +I0911 22:41:19.126764 139895989171968 logging_writer.py:48] [52] global_step=52, grad_norm=0.5, loss=8.58797 +I0911 22:41:19.129662 139924186363072 submission.py:307] 52) loss = 8.588, grad_norm = 0.500 +I0911 22:41:19.558401 139895980779264 logging_writer.py:48] [53] global_step=53, grad_norm=0.5, loss=8.59127 +I0911 22:41:19.561329 139924186363072 submission.py:307] 53) loss = 8.591, grad_norm = 0.500 +I0911 22:41:19.989472 139895989171968 logging_writer.py:48] [54] global_step=54, grad_norm=0.5, loss=8.55304 +I0911 22:41:19.992801 139924186363072 submission.py:307] 54) loss = 8.553, grad_norm = 0.500 +I0911 22:41:20.422389 139895980779264 logging_writer.py:48] [55] global_step=55, grad_norm=0.5, loss=8.5274 +I0911 22:41:20.425262 139924186363072 submission.py:307] 55) loss = 8.527, grad_norm = 0.500 +I0911 22:41:20.853582 139895989171968 logging_writer.py:48] [56] global_step=56, grad_norm=0.5, loss=8.48329 +I0911 22:41:20.856508 139924186363072 submission.py:307] 56) loss = 8.483, grad_norm = 0.500 +I0911 22:41:21.284170 139895980779264 logging_writer.py:48] [57] global_step=57, grad_norm=0.5, loss=8.56201 +I0911 22:41:21.287054 139924186363072 submission.py:307] 57) loss = 8.562, grad_norm = 0.500 +I0911 22:41:21.716800 139895989171968 logging_writer.py:48] [58] global_step=58, grad_norm=0.5, loss=8.4943 +I0911 22:41:21.719694 139924186363072 submission.py:307] 58) loss = 8.494, grad_norm = 0.500 +I0911 22:41:22.148848 139895980779264 logging_writer.py:48] [59] global_step=59, grad_norm=0.5, loss=8.47504 +I0911 22:41:22.151746 139924186363072 submission.py:307] 59) loss = 8.475, grad_norm = 0.500 +I0911 22:41:22.581092 139895989171968 logging_writer.py:48] [60] global_step=60, grad_norm=0.5, loss=8.44683 +I0911 22:41:22.583981 139924186363072 submission.py:307] 60) loss = 8.447, grad_norm = 0.500 +I0911 22:41:23.013632 139895980779264 logging_writer.py:48] [61] global_step=61, grad_norm=0.5, loss=8.341 +I0911 22:41:23.016492 139924186363072 submission.py:307] 61) loss = 8.341, grad_norm = 0.500 +I0911 22:41:23.446172 139895989171968 logging_writer.py:48] [62] global_step=62, grad_norm=0.5, loss=8.41038 +I0911 22:41:23.449133 139924186363072 submission.py:307] 62) loss = 8.410, grad_norm = 0.500 +I0911 22:41:23.878592 139895980779264 logging_writer.py:48] [63] global_step=63, grad_norm=0.5, loss=8.33196 +I0911 22:41:23.881508 139924186363072 submission.py:307] 63) loss = 8.332, grad_norm = 0.500 +I0911 22:41:24.310569 139895989171968 logging_writer.py:48] [64] global_step=64, grad_norm=0.5, loss=8.321 +I0911 22:41:24.313469 139924186363072 submission.py:307] 64) loss = 8.321, grad_norm = 0.500 +I0911 22:41:24.740160 139895980779264 logging_writer.py:48] [65] global_step=65, grad_norm=0.5, loss=8.29759 +I0911 22:41:24.743090 139924186363072 submission.py:307] 65) loss = 8.298, grad_norm = 0.500 +I0911 22:41:25.170791 139895989171968 logging_writer.py:48] [66] global_step=66, grad_norm=0.5, loss=8.30386 +I0911 22:41:25.173669 139924186363072 submission.py:307] 66) loss = 8.304, grad_norm = 0.500 +I0911 22:41:25.604295 139895980779264 logging_writer.py:48] [67] global_step=67, grad_norm=0.5, loss=8.2603 +I0911 22:41:25.607261 139924186363072 submission.py:307] 67) loss = 8.260, grad_norm = 0.500 +I0911 22:41:26.037040 139895989171968 logging_writer.py:48] [68] global_step=68, grad_norm=0.5, loss=8.24251 +I0911 22:41:26.039946 139924186363072 submission.py:307] 68) loss = 8.243, grad_norm = 0.500 +I0911 22:41:26.469934 139895980779264 logging_writer.py:48] [69] global_step=69, grad_norm=0.5, loss=8.17439 +I0911 22:41:26.472856 139924186363072 submission.py:307] 69) loss = 8.174, grad_norm = 0.500 +I0911 22:41:26.902756 139895989171968 logging_writer.py:48] [70] global_step=70, grad_norm=0.5, loss=8.19597 +I0911 22:41:26.905749 139924186363072 submission.py:307] 70) loss = 8.196, grad_norm = 0.500 +I0911 22:41:27.334865 139895980779264 logging_writer.py:48] [71] global_step=71, grad_norm=0.5, loss=8.1626 +I0911 22:41:27.337811 139924186363072 submission.py:307] 71) loss = 8.163, grad_norm = 0.500 +I0911 22:41:27.768069 139895989171968 logging_writer.py:48] [72] global_step=72, grad_norm=0.5, loss=8.23273 +I0911 22:41:27.770974 139924186363072 submission.py:307] 72) loss = 8.233, grad_norm = 0.500 +I0911 22:41:28.198949 139895980779264 logging_writer.py:48] [73] global_step=73, grad_norm=0.5, loss=8.1489 +I0911 22:41:28.201838 139924186363072 submission.py:307] 73) loss = 8.149, grad_norm = 0.500 +I0911 22:41:28.632211 139895989171968 logging_writer.py:48] [74] global_step=74, grad_norm=0.5, loss=8.09388 +I0911 22:41:28.635084 139924186363072 submission.py:307] 74) loss = 8.094, grad_norm = 0.500 +I0911 22:41:29.065753 139895980779264 logging_writer.py:48] [75] global_step=75, grad_norm=0.5, loss=8.16335 +I0911 22:41:29.068670 139924186363072 submission.py:307] 75) loss = 8.163, grad_norm = 0.500 +I0911 22:41:29.498232 139895989171968 logging_writer.py:48] [76] global_step=76, grad_norm=0.5, loss=8.08835 +I0911 22:41:29.501031 139924186363072 submission.py:307] 76) loss = 8.088, grad_norm = 0.500 +I0911 22:41:29.930704 139895980779264 logging_writer.py:48] [77] global_step=77, grad_norm=0.5, loss=8.04184 +I0911 22:41:29.933790 139924186363072 submission.py:307] 77) loss = 8.042, grad_norm = 0.500 +I0911 22:41:30.362908 139895989171968 logging_writer.py:48] [78] global_step=78, grad_norm=0.5, loss=8.06165 +I0911 22:41:30.365863 139924186363072 submission.py:307] 78) loss = 8.062, grad_norm = 0.500 +I0911 22:41:30.795064 139895980779264 logging_writer.py:48] [79] global_step=79, grad_norm=0.5, loss=8.03416 +I0911 22:41:30.797978 139924186363072 submission.py:307] 79) loss = 8.034, grad_norm = 0.500 +I0911 22:41:31.228832 139895989171968 logging_writer.py:48] [80] global_step=80, grad_norm=0.5, loss=8.00398 +I0911 22:41:31.231721 139924186363072 submission.py:307] 80) loss = 8.004, grad_norm = 0.500 +I0911 22:41:31.660629 139895980779264 logging_writer.py:48] [81] global_step=81, grad_norm=0.5, loss=7.94552 +I0911 22:41:31.663506 139924186363072 submission.py:307] 81) loss = 7.946, grad_norm = 0.500 +I0911 22:41:32.093093 139895989171968 logging_writer.py:48] [82] global_step=82, grad_norm=0.5, loss=7.93415 +I0911 22:41:32.095945 139924186363072 submission.py:307] 82) loss = 7.934, grad_norm = 0.500 +I0911 22:41:32.526391 139895980779264 logging_writer.py:48] [83] global_step=83, grad_norm=0.5, loss=7.9515 +I0911 22:41:32.529284 139924186363072 submission.py:307] 83) loss = 7.951, grad_norm = 0.500 +I0911 22:41:32.958187 139895989171968 logging_writer.py:48] [84] global_step=84, grad_norm=0.5, loss=7.91131 +I0911 22:41:32.960993 139924186363072 submission.py:307] 84) loss = 7.911, grad_norm = 0.500 +I0911 22:41:33.391623 139895980779264 logging_writer.py:48] [85] global_step=85, grad_norm=0.5, loss=7.96782 +I0911 22:41:33.394590 139924186363072 submission.py:307] 85) loss = 7.968, grad_norm = 0.500 +I0911 22:41:33.823734 139895989171968 logging_writer.py:48] [86] global_step=86, grad_norm=0.5, loss=7.88392 +I0911 22:41:33.826673 139924186363072 submission.py:307] 86) loss = 7.884, grad_norm = 0.500 +I0911 22:41:34.253933 139895980779264 logging_writer.py:48] [87] global_step=87, grad_norm=0.5, loss=7.87876 +I0911 22:41:34.256865 139924186363072 submission.py:307] 87) loss = 7.879, grad_norm = 0.500 +I0911 22:41:34.686421 139895989171968 logging_writer.py:48] [88] global_step=88, grad_norm=0.5, loss=7.7976 +I0911 22:41:34.689280 139924186363072 submission.py:307] 88) loss = 7.798, grad_norm = 0.500 +I0911 22:41:35.119967 139895980779264 logging_writer.py:48] [89] global_step=89, grad_norm=0.5, loss=7.78565 +I0911 22:41:35.122957 139924186363072 submission.py:307] 89) loss = 7.786, grad_norm = 0.500 +I0911 22:41:35.552831 139895989171968 logging_writer.py:48] [90] global_step=90, grad_norm=0.5, loss=7.75302 +I0911 22:41:35.555739 139924186363072 submission.py:307] 90) loss = 7.753, grad_norm = 0.500 +I0911 22:41:35.985381 139895980779264 logging_writer.py:48] [91] global_step=91, grad_norm=0.5, loss=7.84193 +I0911 22:41:35.988218 139924186363072 submission.py:307] 91) loss = 7.842, grad_norm = 0.500 +I0911 22:41:36.418313 139895989171968 logging_writer.py:48] [92] global_step=92, grad_norm=0.5, loss=7.80588 +I0911 22:41:36.421234 139924186363072 submission.py:307] 92) loss = 7.806, grad_norm = 0.500 +I0911 22:41:36.851115 139895980779264 logging_writer.py:48] [93] global_step=93, grad_norm=0.5, loss=7.74813 +I0911 22:41:36.854030 139924186363072 submission.py:307] 93) loss = 7.748, grad_norm = 0.500 +I0911 22:41:37.285704 139895989171968 logging_writer.py:48] [94] global_step=94, grad_norm=0.5, loss=7.79352 +I0911 22:41:37.288572 139924186363072 submission.py:307] 94) loss = 7.794, grad_norm = 0.500 +I0911 22:41:37.717504 139895980779264 logging_writer.py:48] [95] global_step=95, grad_norm=0.5, loss=7.75346 +I0911 22:41:37.720786 139924186363072 submission.py:307] 95) loss = 7.753, grad_norm = 0.500 +I0911 22:41:38.149924 139895989171968 logging_writer.py:48] [96] global_step=96, grad_norm=0.5, loss=7.67701 +I0911 22:41:38.152797 139924186363072 submission.py:307] 96) loss = 7.677, grad_norm = 0.500 +I0911 22:41:38.582348 139895980779264 logging_writer.py:48] [97] global_step=97, grad_norm=0.5, loss=7.7434 +I0911 22:41:38.585298 139924186363072 submission.py:307] 97) loss = 7.743, grad_norm = 0.500 +I0911 22:41:39.014595 139895989171968 logging_writer.py:48] [98] global_step=98, grad_norm=0.5, loss=7.76799 +I0911 22:41:39.017626 139924186363072 submission.py:307] 98) loss = 7.768, grad_norm = 0.500 +I0911 22:41:39.447905 139895980779264 logging_writer.py:48] [99] global_step=99, grad_norm=0.5, loss=7.66764 +I0911 22:41:39.450856 139924186363072 submission.py:307] 99) loss = 7.668, grad_norm = 0.500 +I0911 22:41:39.881055 139895989171968 logging_writer.py:48] [100] global_step=100, grad_norm=0.5, loss=7.61253 +I0911 22:41:39.883958 139924186363072 submission.py:307] 100) loss = 7.613, grad_norm = 0.500 +I0911 22:44:27.607244 139895980779264 logging_writer.py:48] [500] global_step=500, grad_norm=0.5, loss=5.62951 +I0911 22:44:27.610667 139924186363072 submission.py:307] 500) loss = 5.630, grad_norm = 0.500 +I0911 22:47:57.622242 139895989171968 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.499999, loss=4.55093 +I0911 22:47:57.625794 139924186363072 submission.py:307] 1000) loss = 4.551, grad_norm = 0.500 +I0911 22:51:27.719006 139895980779264 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.480317, loss=4.19094 +I0911 22:51:27.722259 139924186363072 submission.py:307] 1500) loss = 4.191, grad_norm = 0.480 +I0911 22:54:57.777423 139895989171968 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.312001, loss=3.93401 +I0911 22:54:57.780482 139924186363072 submission.py:307] 2000) loss = 3.934, grad_norm = 0.312 +I0911 22:58:27.870755 139895980779264 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.300649, loss=3.88784 +I0911 22:58:27.874035 139924186363072 submission.py:307] 2500) loss = 3.888, grad_norm = 0.301 +I0911 23:01:57.967801 139895989171968 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.280996, loss=3.68137 +I0911 23:01:57.970960 139924186363072 submission.py:307] 3000) loss = 3.681, grad_norm = 0.281 +I0911 23:05:28.011390 139895980779264 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.262206, loss=3.69535 +I0911 23:05:28.014337 139924186363072 submission.py:307] 3500) loss = 3.695, grad_norm = 0.262 +I0911 23:08:58.060926 139895989171968 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.232976, loss=3.50962 +I0911 23:08:58.063969 139924186363072 submission.py:307] 4000) loss = 3.510, grad_norm = 0.233 +I0911 23:12:28.147549 139895980779264 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.21975, loss=3.59591 +I0911 23:12:28.150736 139924186363072 submission.py:307] 4500) loss = 3.596, grad_norm = 0.220 +I0911 23:15:58.226694 139895989171968 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.210281, loss=3.42363 +I0911 23:15:58.229922 139924186363072 submission.py:307] 5000) loss = 3.424, grad_norm = 0.210 +I0911 23:19:28.291715 139895980779264 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.198051, loss=3.51651 +I0911 23:19:28.295031 139924186363072 submission.py:307] 5500) loss = 3.517, grad_norm = 0.198 +I0911 23:22:58.327363 139895989171968 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.187839, loss=3.43879 +I0911 23:22:58.330412 139924186363072 submission.py:307] 6000) loss = 3.439, grad_norm = 0.188 +I0911 23:23:47.205816 139924186363072 spec.py:333] Evaluating on the training split. +I0911 23:24:04.969696 139924186363072 spec.py:346] Evaluating on the validation split. +I0911 23:26:56.114624 139924186363072 spec.py:363] Evaluating on the test split. +I0911 23:26:56.115228 139924186363072 submission_runner.py:516] Time since start: 3030.71s, Step: 6116, {'train/loss': 3.46959228515625, 'train/ppl': 32.123642488533434, 'validation/loss': 3.4614988361173276, 'validation/ppl': 31.86470070464744, 'validation/num_examples': 100000, 'test/loss': 0.0, 'test/ppl': 1.0, 'test/num_examples': 0, 'score': 2611.8812248706818, 'total_duration': 3030.7117323875427, 'accumulated_submission_time': 2611.8812248706818, 'accumulated_eval_time': 414.7203345298767, 'accumulated_logging_time': 0.027541399002075195} +I0911 23:26:56.137340 139895980779264 logging_writer.py:48] [6116] accumulated_eval_time=414.72, accumulated_logging_time=0.0275414, accumulated_submission_time=2611.88, global_step=6116, preemption_count=0, score=2611.88, test/loss=0, test/num_examples=0, test/ppl=1, total_duration=3030.71, train/loss=3.46959, train/ppl=32.1236, validation/loss=3.4615, validation/num_examples=100000, validation/ppl=31.8647 +I0911 23:29:38.694657 139895989171968 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.18351, loss=3.48661 +I0911 23:29:38.697672 139924186363072 submission.py:307] 6500) loss = 3.487, grad_norm = 0.184 +I0911 23:33:08.760053 139895980779264 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.175992, loss=3.43592 +I0911 23:33:08.763058 139924186363072 submission.py:307] 7000) loss = 3.436, grad_norm = 0.176 +I0911 23:36:38.850028 139895989171968 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.163185, loss=3.38235 +I0911 23:36:38.853128 139924186363072 submission.py:307] 7500) loss = 3.382, grad_norm = 0.163 +I0911 23:40:08.956355 139895980779264 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.15939, loss=3.30425 +I0911 23:40:08.959726 139924186363072 submission.py:307] 8000) loss = 3.304, grad_norm = 0.159 +I0911 23:43:39.067935 139895989171968 logging_writer.py:48] [8500] global_step=8500, grad_norm=0.157463, loss=3.40771 +I0911 23:43:39.070963 139924186363072 submission.py:307] 8500) loss = 3.408, grad_norm = 0.157 +I0911 23:47:09.158224 139895980779264 logging_writer.py:48] [9000] global_step=9000, grad_norm=0.153076, loss=3.28958 +I0911 23:47:09.161600 139924186363072 submission.py:307] 9000) loss = 3.290, grad_norm = 0.153 +I0911 23:50:39.282922 139895989171968 logging_writer.py:48] [9500] global_step=9500, grad_norm=0.145874, loss=3.27782 +I0911 23:50:39.286069 139924186363072 submission.py:307] 9500) loss = 3.278, grad_norm = 0.146 +I0911 23:54:09.284416 139895980779264 logging_writer.py:48] [10000] global_step=10000, grad_norm=0.137663, loss=3.28301 +I0911 23:54:09.287517 139924186363072 submission.py:307] 10000) loss = 3.283, grad_norm = 0.138 +I0911 23:57:39.257245 139895989171968 logging_writer.py:48] [10500] global_step=10500, grad_norm=0.1394, loss=3.28406 +I0911 23:57:39.260205 139924186363072 submission.py:307] 10500) loss = 3.284, grad_norm = 0.139 +I0912 00:01:09.200792 139895980779264 logging_writer.py:48] [11000] global_step=11000, grad_norm=0.136365, loss=3.35699 +I0912 00:01:09.203955 139924186363072 submission.py:307] 11000) loss = 3.357, grad_norm = 0.136 +I0912 00:04:39.054793 139895989171968 logging_writer.py:48] [11500] global_step=11500, grad_norm=0.132588, loss=3.34043 +I0912 00:04:39.058045 139924186363072 submission.py:307] 11500) loss = 3.340, grad_norm = 0.133 +I0912 00:08:08.913104 139895980779264 logging_writer.py:48] [12000] global_step=12000, grad_norm=0.129125, loss=3.26222 +I0912 00:08:08.916135 139924186363072 submission.py:307] 12000) loss = 3.262, grad_norm = 0.129 +I0912 00:09:47.697701 139924186363072 spec.py:333] Evaluating on the training split. +I0912 00:10:05.370584 139924186363072 spec.py:346] Evaluating on the validation split. +I0912 00:12:56.371982 139924186363072 spec.py:363] Evaluating on the test split. +I0912 00:12:56.372583 139924186363072 submission_runner.py:516] Time since start: 5790.97s, Step: 12235, {'train/loss': 3.29146728515625, 'train/ppl': 26.882278700775778, 'validation/loss': 3.280596876998082, 'validation/ppl': 26.591639902217743, 'validation/num_examples': 100000, 'test/loss': 0.0, 'test/ppl': 1.0, 'test/num_examples': 0, 'score': 5180.229433774948, 'total_duration': 5790.969086408615, 'accumulated_submission_time': 5180.229433774948, 'accumulated_eval_time': 603.395170211792, 'accumulated_logging_time': 0.057312726974487305} +I0912 00:12:56.392792 139895989171968 logging_writer.py:48] [12235] accumulated_eval_time=603.395, accumulated_logging_time=0.0573127, accumulated_submission_time=5180.23, global_step=12235, preemption_count=0, score=5180.23, test/loss=0, test/num_examples=0, test/ppl=1, total_duration=5790.97, train/loss=3.29147, train/ppl=26.8823, validation/loss=3.2806, validation/num_examples=100000, validation/ppl=26.5916 +I0912 00:14:48.470940 139895980779264 logging_writer.py:48] [12500] global_step=12500, grad_norm=0.125816, loss=3.19922 +I0912 00:14:48.473963 139924186363072 submission.py:307] 12500) loss = 3.199, grad_norm = 0.126 +I0912 00:18:18.086942 139895989171968 logging_writer.py:48] [13000] global_step=13000, grad_norm=0.12033, loss=3.2214 +I0912 00:18:18.090026 139924186363072 submission.py:307] 13000) loss = 3.221, grad_norm = 0.120 +I0912 00:21:47.871868 139895980779264 logging_writer.py:48] [13500] global_step=13500, grad_norm=0.119344, loss=3.17723 +I0912 00:21:47.874928 139924186363072 submission.py:307] 13500) loss = 3.177, grad_norm = 0.119 +I0912 00:25:17.626703 139895989171968 logging_writer.py:48] [14000] global_step=14000, grad_norm=0.118241, loss=3.31098 +I0912 00:25:17.629903 139924186363072 submission.py:307] 14000) loss = 3.311, grad_norm = 0.118 +I0912 00:28:47.319238 139895980779264 logging_writer.py:48] [14500] global_step=14500, grad_norm=0.117236, loss=3.28337 +I0912 00:28:47.322334 139924186363072 submission.py:307] 14500) loss = 3.283, grad_norm = 0.117 +I0912 00:32:17.117357 139895989171968 logging_writer.py:48] [15000] global_step=15000, grad_norm=0.113259, loss=3.14952 +I0912 00:32:17.120329 139924186363072 submission.py:307] 15000) loss = 3.150, grad_norm = 0.113 +I0912 00:35:46.927402 139895980779264 logging_writer.py:48] [15500] global_step=15500, grad_norm=0.109029, loss=3.10191 +I0912 00:35:46.930446 139924186363072 submission.py:307] 15500) loss = 3.102, grad_norm = 0.109 +I0912 00:39:16.695101 139895989171968 logging_writer.py:48] [16000] global_step=16000, grad_norm=0.109891, loss=3.24456 +I0912 00:39:16.698216 139924186363072 submission.py:307] 16000) loss = 3.245, grad_norm = 0.110 +I0912 00:42:46.432888 139895980779264 logging_writer.py:48] [16500] global_step=16500, grad_norm=0.10648, loss=3.16352 +I0912 00:42:46.435959 139924186363072 submission.py:307] 16500) loss = 3.164, grad_norm = 0.106 +I0912 00:46:16.166957 139895989171968 logging_writer.py:48] [17000] global_step=17000, grad_norm=0.103878, loss=3.22431 +I0912 00:46:16.170110 139924186363072 submission.py:307] 17000) loss = 3.224, grad_norm = 0.104 +I0912 00:49:45.929859 139895980779264 logging_writer.py:48] [17500] global_step=17500, grad_norm=0.106013, loss=3.12073 +I0912 00:49:45.933386 139924186363072 submission.py:307] 17500) loss = 3.121, grad_norm = 0.106 +I0912 00:53:15.668147 139895989171968 logging_writer.py:48] [18000] global_step=18000, grad_norm=0.102087, loss=3.22557 +I0912 00:53:15.671164 139924186363072 submission.py:307] 18000) loss = 3.226, grad_norm = 0.102 +I0912 00:55:48.059669 139924186363072 spec.py:333] Evaluating on the training split. +I0912 00:56:05.754373 139924186363072 spec.py:346] Evaluating on the validation split. +I0912 00:58:56.913480 139924186363072 spec.py:363] Evaluating on the test split. +I0912 00:58:56.914083 139924186363072 submission_runner.py:516] Time since start: 8551.51s, Step: 18363, {'train/loss': 3.2044586181640624, 'train/ppl': 24.642155589132592, 'validation/loss': 3.1929507048233696, 'validation/ppl': 24.360201262704976, 'validation/num_examples': 100000, 'test/loss': 0.0, 'test/ppl': 1.0, 'test/num_examples': 0, 'score': 7748.8618953228, 'total_duration': 8551.51058459282, 'accumulated_submission_time': 7748.8618953228, 'accumulated_eval_time': 792.2495367527008, 'accumulated_logging_time': 0.08498859405517578} +I0912 00:58:56.935828 139895980779264 logging_writer.py:48] [18363] accumulated_eval_time=792.25, accumulated_logging_time=0.0849886, accumulated_submission_time=7748.86, global_step=18363, preemption_count=0, score=7748.86, test/loss=0, test/num_examples=0, test/ppl=1, total_duration=8551.51, train/loss=3.20446, train/ppl=24.6422, validation/loss=3.19295, validation/num_examples=100000, validation/ppl=24.3602 +I0912 00:59:55.520708 139895989171968 logging_writer.py:48] [18500] global_step=18500, grad_norm=0.103941, loss=3.12877 +I0912 00:59:55.523737 139924186363072 submission.py:307] 18500) loss = 3.129, grad_norm = 0.104 +I0912 01:03:25.097697 139895980779264 logging_writer.py:48] [19000] global_step=19000, grad_norm=0.108919, loss=3.18669 +I0912 01:03:25.100942 139924186363072 submission.py:307] 19000) loss = 3.187, grad_norm = 0.109 +I0912 01:06:54.929475 139895989171968 logging_writer.py:48] [19500] global_step=19500, grad_norm=0.0981462, loss=3.15186 +I0912 01:06:54.932518 139924186363072 submission.py:307] 19500) loss = 3.152, grad_norm = 0.098 +I0912 01:10:24.708503 139895980779264 logging_writer.py:48] [20000] global_step=20000, grad_norm=0.101852, loss=3.04596 +I0912 01:10:24.711560 139924186363072 submission.py:307] 20000) loss = 3.046, grad_norm = 0.102 +I0912 01:13:54.518823 139895989171968 logging_writer.py:48] [20500] global_step=20500, grad_norm=0.0998804, loss=3.18407 +I0912 01:13:54.521867 139924186363072 submission.py:307] 20500) loss = 3.184, grad_norm = 0.100 +I0912 01:17:24.334611 139895980779264 logging_writer.py:48] [21000] global_step=21000, grad_norm=0.0981251, loss=3.1658 +I0912 01:17:24.337704 139924186363072 submission.py:307] 21000) loss = 3.166, grad_norm = 0.098 +I0912 01:20:54.164061 139895989171968 logging_writer.py:48] [21500] global_step=21500, grad_norm=0.0915616, loss=3.09324 +I0912 01:20:54.166954 139924186363072 submission.py:307] 21500) loss = 3.093, grad_norm = 0.092 +I0912 01:24:24.026594 139895980779264 logging_writer.py:48] [22000] global_step=22000, grad_norm=0.0948603, loss=3.16961 +I0912 01:24:24.029630 139924186363072 submission.py:307] 22000) loss = 3.170, grad_norm = 0.095 +I0912 01:27:53.807242 139895989171968 logging_writer.py:48] [22500] global_step=22500, grad_norm=0.0916049, loss=3.18716 +I0912 01:27:53.810465 139924186363072 submission.py:307] 22500) loss = 3.187, grad_norm = 0.092 +I0912 01:31:23.544998 139895980779264 logging_writer.py:48] [23000] global_step=23000, grad_norm=0.0921158, loss=3.2415 +I0912 01:31:23.547962 139924186363072 submission.py:307] 23000) loss = 3.241, grad_norm = 0.092 +I0912 01:34:53.256163 139895989171968 logging_writer.py:48] [23500] global_step=23500, grad_norm=0.0993161, loss=3.24387 +I0912 01:34:53.259048 139924186363072 submission.py:307] 23500) loss = 3.244, grad_norm = 0.099 +I0912 01:38:22.998920 139895980779264 logging_writer.py:48] [24000] global_step=24000, grad_norm=0.0904274, loss=3.09541 +I0912 01:38:23.002022 139924186363072 submission.py:307] 24000) loss = 3.095, grad_norm = 0.090 +I0912 01:41:48.685955 139924186363072 spec.py:333] Evaluating on the training split. +I0912 01:42:06.347743 139924186363072 spec.py:346] Evaluating on the validation split. +I0912 01:44:57.182804 139924186363072 spec.py:363] Evaluating on the test split. +I0912 01:44:57.183420 139924186363072 submission_runner.py:516] Time since start: 11311.78s, Step: 24490, {'train/loss': 3.1459590911865236, 'train/ppl': 23.24195594217749, 'validation/loss': 3.134155585637788, 'validation/ppl': 22.969232101313764, 'validation/num_examples': 100000, 'test/loss': 0.0, 'test/ppl': 1.0, 'test/num_examples': 0, 'score': 10317.612355232239, 'total_duration': 11311.779911279678, 'accumulated_submission_time': 10317.612355232239, 'accumulated_eval_time': 980.7469322681427, 'accumulated_logging_time': 0.11410927772521973} +I0912 01:44:57.207840 139895989171968 logging_writer.py:48] [24490] accumulated_eval_time=980.747, accumulated_logging_time=0.114109, accumulated_submission_time=10317.6, global_step=24490, preemption_count=0, score=10317.6, test/loss=0, test/num_examples=0, test/ppl=1, total_duration=11311.8, train/loss=3.14596, train/ppl=23.242, validation/loss=3.13416, validation/num_examples=100000, validation/ppl=22.9692 +I0912 01:45:02.697284 139895980779264 logging_writer.py:48] [24500] global_step=24500, grad_norm=0.0911131, loss=3.14831 +I0912 01:45:02.700088 139924186363072 submission.py:307] 24500) loss = 3.148, grad_norm = 0.091 +I0912 01:48:31.953239 139895989171968 logging_writer.py:48] [25000] global_step=25000, grad_norm=0.087336, loss=3.1186 +I0912 01:48:31.956254 139924186363072 submission.py:307] 25000) loss = 3.119, grad_norm = 0.087 +I0912 01:52:01.634495 139895980779264 logging_writer.py:48] [25500] global_step=25500, grad_norm=0.0872186, loss=3.05995 +I0912 01:52:01.637486 139924186363072 submission.py:307] 25500) loss = 3.060, grad_norm = 0.087 +I0912 01:55:31.356534 139895989171968 logging_writer.py:48] [26000] global_step=26000, grad_norm=0.0964388, loss=3.04478 +I0912 01:55:31.359647 139924186363072 submission.py:307] 26000) loss = 3.045, grad_norm = 0.096 +I0912 01:59:01.017215 139895980779264 logging_writer.py:48] [26500] global_step=26500, grad_norm=0.087137, loss=3.02721 +I0912 01:59:01.020203 139924186363072 submission.py:307] 26500) loss = 3.027, grad_norm = 0.087 +I0912 02:02:30.767241 139895989171968 logging_writer.py:48] [27000] global_step=27000, grad_norm=0.0831365, loss=2.99767 +I0912 02:02:30.770340 139924186363072 submission.py:307] 27000) loss = 2.998, grad_norm = 0.083 +I0912 02:06:00.556016 139895980779264 logging_writer.py:48] [27500] global_step=27500, grad_norm=0.0857155, loss=3.13677 +I0912 02:06:00.559036 139924186363072 submission.py:307] 27500) loss = 3.137, grad_norm = 0.086 +I0912 02:09:30.291144 139895989171968 logging_writer.py:48] [28000] global_step=28000, grad_norm=0.0931385, loss=3.14298 +I0912 02:09:30.294214 139924186363072 submission.py:307] 28000) loss = 3.143, grad_norm = 0.093 +I0912 02:13:00.083648 139895980779264 logging_writer.py:48] [28500] global_step=28500, grad_norm=0.089762, loss=3.1517 +I0912 02:13:00.086574 139924186363072 submission.py:307] 28500) loss = 3.152, grad_norm = 0.090 +I0912 02:16:29.835151 139895989171968 logging_writer.py:48] [29000] global_step=29000, grad_norm=0.0879644, loss=3.0691 +I0912 02:16:29.838304 139924186363072 submission.py:307] 29000) loss = 3.069, grad_norm = 0.088 +I0912 02:19:59.533909 139895980779264 logging_writer.py:48] [29500] global_step=29500, grad_norm=0.083369, loss=3.04432 +I0912 02:19:59.536926 139924186363072 submission.py:307] 29500) loss = 3.044, grad_norm = 0.083 +I0912 02:23:29.291051 139895989171968 logging_writer.py:48] [30000] global_step=30000, grad_norm=0.0854799, loss=3.02347 +I0912 02:23:29.293978 139924186363072 submission.py:307] 30000) loss = 3.023, grad_norm = 0.085 +I0912 02:26:59.043364 139895980779264 logging_writer.py:48] [30500] global_step=30500, grad_norm=0.0846618, loss=3.11224 +I0912 02:26:59.046449 139924186363072 submission.py:307] 30500) loss = 3.112, grad_norm = 0.085 +I0912 02:27:49.107970 139924186363072 spec.py:333] Evaluating on the training split. +I0912 02:28:06.800046 139924186363072 spec.py:346] Evaluating on the validation split. +I0912 02:30:58.100536 139924186363072 spec.py:363] Evaluating on the test split. +I0912 02:30:58.101159 139924186363072 submission_runner.py:516] Time since start: 14072.70s, Step: 30619, {'train/loss': 3.099249076843262, 'train/ppl': 22.181288582778382, 'validation/loss': 3.0879880314897696, 'validation/ppl': 21.932905238908088, 'validation/num_examples': 100000, 'test/loss': 0.0, 'test/ppl': 1.0, 'test/num_examples': 0, 'score': 12886.524369478226, 'total_duration': 14072.697648763657, 'accumulated_submission_time': 12886.524369478226, 'accumulated_eval_time': 1169.7400870323181, 'accumulated_logging_time': 0.14608383178710938} +I0912 02:30:58.123644 139895989171968 logging_writer.py:48] [30619] accumulated_eval_time=1169.74, accumulated_logging_time=0.146084, accumulated_submission_time=12886.5, global_step=30619, preemption_count=0, score=12886.5, test/loss=0, test/num_examples=0, test/ppl=1, total_duration=14072.7, train/loss=3.09925, train/ppl=22.1813, validation/loss=3.08799, validation/num_examples=100000, validation/ppl=21.9329 +I0912 02:30:58.660858 139895980779264 logging_writer.py:48] [30619] global_step=30619, preemption_count=0, score=12886.5 +I0912 02:30:58.669167 139924186363072 submission_runner.py:857] Final finewebedu_lm score: 12886.524369478226 diff --git a/logs/self_tuning/ademamix_golden/study_2/finewebedu_lm_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_2/finewebedu_lm_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..5615bf5e --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/finewebedu_lm_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,7 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/loss,test/num_examples,test/ppl,total_duration,train/loss,train/ppl,validation/loss,validation/num_examples,validation/ppl +225.8109667301178,0.0,43.54396414756775,1,0,43.54396414756775,0.0,0,1.0,269.9600627422333,11.046409606933594,62718.36628502924,11.04719469309463,100000,62767.6249400565 +414.7203345298767,0.0275413990020751,2611.8812248706818,6116,0,2611.8812248706818,0.0,0,1.0,3030.7117323875427,3.46959228515625,32.123642488533434,3.461498836117328,100000,31.86470070464744 +603.395170211792,0.0573127269744873,5180.229433774948,12235,0,5180.229433774948,0.0,0,1.0,5790.969086408615,3.29146728515625,26.88227870077577,3.280596876998082,100000,26.591639902217743 +792.2495367527008,0.0849885940551757,7748.8618953228,18363,0,7748.8618953228,0.0,0,1.0,8551.51058459282,3.2044586181640624,24.642155589132592,3.1929507048233696,100000,24.360201262704976 +980.7469322681428,0.1141092777252197,10317.61235523224,24490,0,10317.61235523224,0.0,0,1.0,11311.779911279678,3.145959091186524,23.24195594217749,3.134155585637788,100000,22.969232101313764 +1169.7400870323181,0.14608383178710938,12886.524369478226,30619,0,12886.524369478226,0.0,0,1.0,14072.697648763657,3.099249076843262,22.181288582778382,3.0879880314897696,100000,21.932905238908088 diff --git a/logs/self_tuning/ademamix_golden/study_2/finewebedu_lm_pytorch/trial_1/events.out.tfevents.1789166185.b93e232cdf61.38.0.v2 b/logs/self_tuning/ademamix_golden/study_2/finewebedu_lm_pytorch/trial_1/events.out.tfevents.1789166185.b93e232cdf61.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..98b1b9a4cfb89f77367b7705987fc53fc0ec53a6 GIT binary patch literal 39308 zcmbW=c|Z;A9tZGHagLBBYnI4bijb|6rK~AgN~EF`+O$|A^%AmYEnBihL|GEDB%)-= zp2${`gsf%1Z)ft%eCK(_n|uGg_xHW?ncqCm%82Jj*?k+= znCRm)#Z%90hKr|PF9U0Bc}qVRPhT&e$?je=^(J}wxaj%&O>x(ADf{nz9cKDW@pJK6 zQSWGX-SV`L2d|jP>T4M7E=%j6C2uZQcK7o2?I*8Rsg6=hWrK`PeTP(+%j8vkCpo!0 z`S_+Bo>$npymsq?1uta=&C6@s$hGA%S05*52Tv~_kKX~me*H3mfWapWl)97$sO@Pn zMs_T?JfNMtjl8<6yVpb~cL!fT7w_K@W&dM{=y{+|rLN@>6S_1QFB`KdE2r%E)hl&q zD{n2A`<0y&eW9b_8IhytU`Bm7jE}oWhC;$+cze75K8y)J{RYBe+)CYhsAu_MEUVi6 zoh&w%MCi&p$ZO4Ta-ZVt+skVTf&>Q? zrSFdiRQq%5R(;EJlwwl$%fe2PoUU>Mc>_U?r@x1T%Pc1kZ+92cY$dDn7+7U=@>@7b zkI$1p?j*VR`oVcF`+tX%G`VoVIfL?J(rz@SNOtrxiD)fvC9mdJ79mh6DFyd|1f{oc zi+YzQG;S3;URM1kNzjw)%4;hUm>%)LXk_@M>6b@&*f+J3Wo415mhu*I72mQI^Sj|H z9rgObku1(Q;SPPe`IwX|`sC;3_wTjq?C+y!`tO|FuZ#LaPF)YBu4d(j@haafUKW^7 za(c)OP4f5fcX#r0adz9dV@aEXX8p?JW`FS9CmZvL#C4N*kv9~^xqG>~PVsbw#IjmY z-zrhL(XBl3ZmNC0?8JAHD82MW7sHa_9<_UwhtyeL_)PYw%KhIzzO)+g0$1$9#j_&K&QC{8yc z5z-1%RGeAYY8xm`5Y|6;Y^2Kk*?F z6fvK#a|%^jF>jGBD!%*X=78eGPf}e$#V_01si25c&HQ};(u!nMoYYvk5EPonUU4B+ zX@&jE4yb5<@54e+j6OrE*$Qf1jCmem2#U=YNrbfGiAdox+7J|Py{bM|?6lQrh3V|} zs0bhWcois0W|Ha(D%xL(SqX~1!6ZUjp@hd_9XYHcD5f`Q#VKU8qS&o1R!6*g9|8(> zZBku9#iI6?PJ&{iE{TvZT|n{u5~;4BV%ogb z??CY;lSD`>THtXs4;dZ;ibZ~QoT56d2x!&{s}EY#90`h^fux$P_%7a~P8jC421P*# ziI7%^?!Hm7UTZ-Sq(6gGsL_fck45J|f7Ti`P|WB~s@V$a2}Jjp&sb3G>_Z}?6{dJL zHQyUpgCgSWYEDsuR)pj=!|EZ;(=tGD?h>hHE2ur{%md5$poqLqBBT{ocpUM|@9hJH zdcXlrQIl3=f6&D0p?CY~gW~l+q?)au?s}asJ#Pkz+Dl1z2s9(!=%BQ9_iaNBy@oWRE-agx8G$<|& zA=PZfSFxhj*bBEoaoL(gNGrndI667kS_F#9@4It~y0qdyZd) z#a>j%Y;N=bMe3!coT5IhIM-67IM`lnZ@}UtZ4(q0H ziiWhJem#*wuVdseP$YCE)ocaTi_VS<)`LRXm_$e`l2LKEU0whv8r;g^6pd&_c=fti zU8~WAC{P4FAk}O|sd!zCoVdmY6s5T&LRyiA$C3Z&lMyIZ`hVdRjcLWibdh3xAKh3` zm>RKUs6HgH+~qWXi6)Ftq>_Rk9iLSMW{2WW-F*Gs#AQtCn#pPlL%== z0V;xSsI~>gfN!pxq8Y8YU00;&VDqvCDDsuvepj;<)aqWaGifp?o~n@uX+<$AnpE4> z1Qb7#7I4SWoK_5KBvK^XJ}Cr6jSHl@f{IE5yS)d+uM83)ttdrBW!vyLP^_P{jZ?Is z73Rk3Se+fv>l-N6O(WH81$9N~+Fo-2g`+QtkX9(+mr?4iOs9jQYV{;ep+ze^Yt%-? zF7>l%pvbF3s@V$ain`K&pB*S_G$9ev3T0F{R#uGxMYn{zoT4SI800AG#jaCtUW1~} zX;RHrP){IREneVG8<0vOq!rar@k#Bh11Kh$z2_9IXhny~wXheBmX*eU;+hqyuAriF zdg5E?MW7vtkXERnVzA9RCr})ETH~4GuBS~arcDzmG+VqxMU_HQ%~nwNi=n2;Z9viK zJ&BN3sH5Wh_6G@|n7O?@r)W(pIvuWwy;$NF`T%-yH-=QR71X*Y*tCBxDE1s85z>l! zsF*w1?g=PHjW**HZD_@kV39(8Ru%w?KzmZnR#59=Sgo1ipqS}GBBT`!QBi$$8h#%2 zp}>(-w51gV&NZ+Xxmz<9f#TQ)Qq5L;7C(Xb-+N;WiUr?DgtS5v6~}MnszER8cg^J# z?P!IOzDQx#XD@y>I53t}vlUb?UM_kb3X0B0Nrbeb1uCva|KkD*qu~*pqCKrpxgt{3 zo#}?(PO6O|)fH51-cn5)6vHQw2x)~jD!j&AUj~YRpGP=F2U;<*ib!$I!}}9lQ5RH? z{;p;#sMT$-Iu^gzOR7a8q!sN@VeFE61{5uKU+0cPhgK}gs*dYow?V{3P&gkT)fH5D z_3mj2y{K}GL`W-iQK53VnLQ}38NT8ay0pT1ok(%%#QY*qROv&i*$Qf1cn+@h3>0+- zk_c% zMMCRs$)KpVswJn;qZN^ZM2eq%JK?u59V19}1r>P{gF1jhbvucWR`fuHd%o#PP^fk= z;S~C`;`Vqst_$nNf%tv*2m?~hR#2<^oTnRpdN98iiI7$pp<=j^N)1pbKCK1w#VW_xqqu(@8q;yT=6kTb>NH>w9&BdZ)(2H0jQq5LS_Y2kd^uwU2Y(^ra z6;`N__xXr_9~FHwms5136}SCG3eD7HIVkjVNHtqQeU}v2v412eoS%~jX~ig1ENwow z67*t_=PyoSNGtl?uZCAt^SA~*Kw&wHRI?S->W({`83c;H^GSrX!WI>aJ15{@+`Vev zBu{bg?oKOudy5p0o7;T>MNV5%%~nwB!onn06%^O?NrbdwJSwKH3K|N%us&hPDSFU~ zHAz*m7q`v!)&+(AIa1A5P`#+Pxb{m>Oi3pZ(h5gZOzZo$IVf%qwB{5&X~oYUqF!WW zbit>=C`(e!R#2+&@iI7$}qhhw9-f~dvF7e-Iq57YPDGPxwt`w0 z{gZFvH)2;0k_c&qJ1W%2+V22`wfSyNVMHqo+K3dXqiTkNVv7~2uAstus8KEGMXW7} zkXCr3Lc7%fA5gseoXjbVX~ii&6jtQKEY8_W?*N{7{ke zvQ|S-6h%MeLQH5yQJ5+!lQWl)?fsry`U*E6LR zH(QDn)6XfP;?XZs%~pI8f2wAG)ZHHxX;tg}J^*P&Fe;MX%JHvRhR)RILKr_oD^5|y zE2@2yu|1&|lY&S!TS4`rz&6MQ6oVF%2x-N9R3zM~jGxoGs|@87{pen7e<)Iv-u!wW z6d^T8HCsW=7v{k;RY37YgG5LxLQtW4Bj+bvQHxV2bBg}7qH70{VvfnQnxOE$N~+lk zsuwQ~G)BeNJ0wC{u?Q7u%k}Gm;?#`!oMHg281u9WUQw!Dbn$-iavrH>E2!`FOuh{b zgI*k3L?WaWp{Te}GJgpuD(ggY3NukdyqExr@ZKas zTCo%rM&pm+x07cco!}Jav?8IUs24#Y$9jRn_64bCE2v(~GrsQ*3gcoDA+1=Bil(my zjeuU*FS^4i2GWY38%4cveE%4KBbL3AR98@u8r@$96b|7eLRzsB6|HBdtOA9b={rs_ zh*tDHA?n2g^`%#!7j}b4HCsVlQ7s#-4gf_XNzEH-`HH*VU|Nyu zB5eNe2*B@xn!HK;H@me><|(R*b(PBDa5ED5fJtJ}Mv zI)1PBYdxuEE2vKo7Ff=`1d7*NNrbdw9V#xb-G_fIU*BQ?rx;2r8g3Q!qNHyI{uah% zG^u7QsQDtUxl>zEtgt5$(uxhJs5d31FZAMMsRO4NMl15Ch>+)q?8fioMJew2>C8jB!&)ghF+-Ik!rSr>c#W=Yj9%SH<3h0 zE4HBGu}g7NxT5mD9Oe|3wBp4wk>YhhCH(&4L!~3XtJw-_kJ{xNd={=K8##%PRz#v= z&a2MfK_NeVjXMr2S}{Uhr1){tqZ{<1e=4bFE2w+-zG=7d6HcwGBtlvdg^Kelro01% zt3x5D7(pvy`~Jk=EC%GS#&5(Xx{_+Pf?5}C(>CC@llERDLRzr{6`c>N`JzHi^@ZZD zXH6?s=lu{Vo;_BAUij1_)ocZIzi4~9#sIjYY&A)Qv|<-3HarbV1I3gyElx3#R_yf= z^YpeKotR_sGXdOOjl&5_srIK>!RQFrinTo;{}4fF-Y?krNxR#3edY_k*p zZgANX5+SXKMa7vq;kM9=qqbq3!iH94G!rQ-i>9pxMO`OS%~nwBB50q#74+hZ8;Ous z97IJ%E{hC<;H}JU%m* zQ`pgpOASl07qjc>;rHD&=aXu-g1UGA8ZB=DiU&(bgtX!)Dmu6kabgI*lgC)I2PwMSL2w=em;!X~i*AH1r?P2NYX!8W$+;-Q#J+sdUj5 z^=#rB{Eqe9b5dPFMW4xCc7tNmYZ4)?IF5<|E9cAtMeQ&{PGL_gqKiwg7gpZJ_EVc`@riQD-}ekXD>T#eN%G{N+Nlqcx|PKr5Cyi+a)Joe}=JYK|MJW-F-G z9hrO45ft;iNrbfG6e>E#53B{fXsGJRDI92pqo!zePuJXp_ls8Qq?)audZA)vgP$Jk zZA2oZ6=zT}Fgo886j$Sxa0*9SQGeejyrMp6Ou%=u+)gCbYz4KtT|XQ-3ca{~mPAM^ zl2LJ8*9|{C_{V-Xr*NVbZ_`EV;_Av>?O=}zaUs=g1$FN(3<}Z%MT!TBkXEFiB7DV< z-O!6W)y{E>iL}CQt@w)SZjGNdA5x2ECY`L8{pbsuvv-{nvuR{1%CjR$N3y&e|AfP!!H8--ND6=8bbze-#+S1p`C`bf$&pC_8LHw&4nZ0cFaX}JW|bDU9U#A& zznP8sM_Y3dLZ)ACu?8S!9r$uKM7QV@zVvhMgjewZu`u8>Bq6~`2+^seH4Y#_ z52nW}AeoH!n`B>{gOI}YpL_taIgc-AL$;sUh7c=@+|B^eEaEdHAxd8nVr6B15+HW> zO7awt3yc?1xZIqDkW3ZJ-2kcalrLvPV%}^-$Xq9twg4$9;4>s4rkZAOX=Q%$iU!D) z>hl`4EL-fy8E+8Sw0Rmr+H~n=2av`Zd^sDE@Msl6>NNG$21uzUpCJhe?v0QUvfKoK zyu7J2Tmgw?{N(#UuscGEGy7TsME@aQ&W05BS%eUS7LBq1@-~;xkc1@LAtYnqLSKL+ z{8}(y0f}P#Mt9p*M}$~~MmzwBjVy$evmq9%JrQ!))8sKg8r0@9Bq2%x2#K~C(;gtU zD`bfZ$U4T4D#x^%j1ZYcsu4h{g!AQWh`y>5LTrBqIRWHgB%dJ(iCT`3;`hIv0c2m~ z%2x_VDC1Xfmo>*B#C+U6TqD=^@a1gCkC8SAahZAhAVBit_zX#i=@x{y4@Me65BqTTqA!`pk8x0R^ z$MdR%DIj_JQg6b(T4Mros_a`B2%I~w`49z%dKLDTc}n=uclVbp3|`B7@*|(|r*~l+ zEV0L{wYw&dvYzHtd-mZ{y)a6;|UD zxfAuDufu-OQx`u;cvzFqsM!0k$L62tXjJ}mo0hoKTJ}SOkNC?MVw;~$Oz2dewfXBUVwwnjLnB`<+SAM|L{ueNz;rm==8YfOU%m;#HZVd-LmuT_@Iiu z6Pr4`E%VfLOv_W-HTbw*W~$Gp{`X63xYWXG6wz?~0I8wf^`>_x>86Aqh#&M#!RmTEzgVJpY)M zcG-RR3gbr!{ z`^^su$QH&Q*4?`_0wEd?j@AT-vngNBhBSB7KuDFsBdY--AH-)!LcHrCWY&|t4gmQv z+`3)svc(?8_)Dmpnxc+$+MT)qAX>J3IUBODL=7SNx_*HGx!}NONJ30C5%SJm%@QET z!!pM!AR&xDU1_Q(g7o^7o&b=B8~JiJq^2x;AGV<WA@N-h67yO43tU<+ zmiGFpfIMM*Cuy2yg^;W3YMueeqP2WE8)Dv586k~#Ucz1L^(H<;5)#}SA(QVYXT$vV z$hc9zHf4)F!$E3(o4iM~QnGG^;lGAtJdh7jaH#pMpyoXsmR78I^91I%dlr1gpXN7< zPokYvb~P*#CIXF7e9|B0w}VNyMTgY>=}nm5zS;3X|JVFh-1+-D=zFFkpU9o4|D4}` z&Q-$qG9RAIXH;x{JNrX70p_;_9(=@K&Tn&PJsb`5Tf8ryMbB?{pKiwonX#|@|e!I1hPyO%tZOHxl___OVkE!=y zetWf?&#Tbu8iB*3* zCw}#P%>mMT|H3s2NSX`%eQ1EraD+r|(K80fm7{z)8#3AJGdB0y>F`wmsdI|Ykc6}t zijc)iKV1RH{1+t`6_8U?Xh^cnP=sW86yVoZ4#j*q8!~t5M}$}ueXIhIm=Zoi5>jl9 zkfLeoJ^+#RaQmi!#CywJ}jgHiw&y6Q*cyVV?92qL+Y^`8^nM7x)-V8?Z6BM7P3NO#xPt$}g*Hsv_@U6Jgtt{~$t zXF7-6d1K&TSgzki5JnGm&)dAmS!}7#yvNY9{yhZ&f0*j3N2=g8-KuT!s)6OMljpx} zmM!cfNUG>qM;+ku%0-pn-zk1=CW!s-xvsR9InH(V$4lP~C_m2rWA!%4t`8FgR%oyb zu6Yo*g<-Q!&V_9ucBCLva$A_O3QwWmvrkiD+E=y{#Qfc8XR%>$4s4f~J)Tx>Uv^8r z5XRUpPa6-y&Ba#zNL|=2m1hX*+1q96BhhW_Om=_+oR=kmf)L5`5*mi*SZ-VrU1OiMlOn)!)J9OxwlrCJ1uZF`4(Ef#!9frrX=xXQmgK2}-@rBY zB}7Bkp=|Ak?qXP`Z4o_=x4$UIf5k+3rJ$Z|x%DXxE%#o>W`O1WdO?V!MST}qJi`jV zg2g>zsjsw-54xe_6kBIEv0+VQo4Of z53rmMy6z^wPGP6Z}k!{n)cP~D?E2w8%nwh7dW$@`~J;1W-ksw6U@?<|+=8qiH z87xy$eWMkYyf}u%=c{PF4Bp+a7}m>%>wDrBp z!4mZKS+>HGb%bF#{^Y1Iwu@xqLm_(rM2r?8^kLslH&TT3rw#X^B6AmhnA4 zwg*eMK&x6hWo!RRBEzz9v&gcxyVqr~d|4={XIo4rXQ1WwikJ|v*sc(SNLnHi(GoW) z&k8J&)sp)wEXhd>i%ApFCp`_<{=l~?=+qI^vn^E&PM~G-e2tl4sohu*B54UuLd$2R zVEmoQ2v>_}g=ODa$&t^+Ms)Z8$Dw8m6b;XWk?;9dK^Q&q`5J18uEhE0 zm7!+~b_oLhF!D7&+X<7@7S_88BcIhiK~hCWKI#B#j7Z)DBj53OLF|8zeAo2-apc=H z(Axz@zUV|jV1-7$7KdNrw&1_p*9x|UV`l`JlH0-3$A9P!BcEQXAm;BzzPJ`a z9bmgWpBTGGvBezycZ<=IH zcwWY9``g2L(YPcCkvuPMqVw`GY6d=z_tyOJKw;T?o?+1q7kzR$siQu=7U+hypq_0p z%sGq}*Nv;!gT+->5F%-@I*&b$^J%*a`Z6iOO!(w)~KR&{i955OV`&idA zf_k=P&o+_8O=}xIDqTwzgh*O?TtbW7#x4Xb^X@1`TET&!i&Q@4Lt}`rmPxr;XB)NOx>nl=? z1odpo6F*U342OkSf~E5SL5QTK+I6%nEzNEWmLh{g2Nagln+(hO`+d;TCEv3zTw`lY z1odo-Z~uMR7cHmi5nypO7lcS!if^K2z%`p#u$aWRdZw@x-DOyEl0^?Cn;ajW0?W_i zf_kn>V?ZfXV9{IK|wXJ{BuaGegmr+9hx=u7WsDVlm^Yk$`phUJN#=o#kW+pGBT zsQN2GJ=@ZH(gD2225!6;1D4_M1tF4_L2uBKutIeQd}C>-9BQkuY_zBaNG4dh)` zjY@~^tgj(RQHZGVb)fGdoD`c>YP1f<*U9w+A%7ZQH*ObwIOyq?@)ky+6O9E?e;8lG zXT;$_rMh$oh4FQpmLTc>8ea!wkHU8udAAirawqFQ$JaGu1_Z$P+Nq-;q+;Xif#yBFOm5@pZq;9KUt@*|V*h)5jqAP!$5(6R%Stf5ULGk3tkC$X6fXJ!N@3Ge zX|OFUwG(7YZVQHQ@f0T7$&F!rt?ML+`MdGedepK#aAgEe*vD@%6}U2{oZM{(rkRrk Z0TnjAJ2gfNOv^k3F@I<3{Q0i!{{Y_ah{pf` literal 0 HcmV?d00001 diff --git a/logs/self_tuning/ademamix_golden/study_2/finewebedu_lm_pytorch/trial_1/flags_0.json b/logs/self_tuning/ademamix_golden/study_2/finewebedu_lm_pytorch/trial_1/flags_0.json new file mode 100644 index 00000000..334e8c3c --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/finewebedu_lm_pytorch/trial_1/flags_0.json @@ -0,0 +1,62 @@ +{ + "logtostderr": false, + "alsologtostderr": false, + "log_dir": "", + "v": 0, + "verbosity": 0, + "logger_levels": {}, + "stderrthreshold": "fatal", + "showprefixforinfo": true, + "run_with_pdb": false, + "pdb_post_mortem": false, + "pdb": false, + "run_with_profiling": false, + "profile_file": null, + "use_cprofile_for_profiling": true, + "only_check_args": false, + "op_conversion_fallback_to_while_loop": true, + "delta_threshold": 0.5, + "tt_check_filter": false, + "tt_single_core_summaries": false, + "runtime_oom_exit": true, + "hbm_oom_exit": true, + "test_srcdir": "", + "test_tmpdir": "/tmp/absl_testing", + "test_random_seed": 301, + "test_randomize_ordering_seed": "", + "xml_output_file": "", + "experimental_orbax_use_distributed_process_id": false, + "experimental_orbax_use_distributed_barrier": false, + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "workload": "finewebedu_lm", + "tuning_ruleset": "self", + "tuning_search_space": null, + "num_tuning_trials": 1, + "data_dir": "/data/fineweb_edu_10B", + "imagenet_v2_data_dir": null, + "librispeech_tokenizer_vocab_path": "", + "framework": "pytorch", + "torch_compile": true, + "experiment_dir": "/experiment_runs", + "experiment_name": "submissions_a100/ademamix_golden/study_2", + "save_checkpoints": false, + "save_intermediate_checkpoints": true, + "resume_last_run": null, + "append_timestamp": false, + "use_wandb": false, + "profile": false, + "max_global_steps": null, + "overwrite": true, + "hparam_start_index": null, + "hparam_end_index": null, + "rng_seed": -965294040, + "set_pytorch_max_split_size": false, + "pytorch_eval_num_workers": 0, + "capture_jax_trace": false, + "skip_evals": false, + "?": false, + "help": false, + "helpshort": false, + "helpfull": false, + "helpxml": false +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_2/finewebedu_lm_pytorch/trial_1/measurements.csv b/logs/self_tuning/ademamix_golden/study_2/finewebedu_lm_pytorch/trial_1/measurements.csv new file mode 100644 index 00000000..b316a8f9 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/finewebedu_lm_pytorch/trial_1/measurements.csv @@ -0,0 +1,170 @@ +global_step,grad_norm,loss,train/loss,train/ppl,validation/loss,validation/ppl,validation/num_examples,test/loss,test/ppl,test/num_examples,score,total_duration,accumulated_submission_time,accumulated_eval_time,accumulated_logging_time,preemption_count +0,0.4999999701976776,11.04960823059082,,,,,,,,,,,,,, +1,,,11.046409606933594,62718.36628502924,11.04719469309463,62767.6249400565,100000.0,0.0,1.0,0.0,43.54396414756775,269.9600627422333,43.54396414756775,225.8109667301178,0.0,0.0 +1,0.4999999105930328,11.04730224609375,,,,,,,,,,,,,, +2,0.4999999105930328,11.02553939819336,,,,,,,,,,,,,, +3,0.4999999701976776,10.992456436157228,,,,,,,,,,,,,, +4,0.5,10.937305450439451,,,,,,,,,,,,,, +5,0.4999999403953552,10.874153137207031,,,,,,,,,,,,,, +6,0.4999999403953552,10.782835006713867,,,,,,,,,,,,,, +7,0.4999999701976776,10.681973457336426,,,,,,,,,,,,,, +8,0.4999999701976776,10.59210205078125,,,,,,,,,,,,,, +9,0.4999999701976776,10.463119506835938,,,,,,,,,,,,,, +10,0.4999999701976776,10.342330932617188,,,,,,,,,,,,,, +11,0.4999998807907104,10.194940567016602,,,,,,,,,,,,,, +12,0.4999999105930328,10.054513931274414,,,,,,,,,,,,,, +13,0.4999999701976776,9.930789947509766,,,,,,,,,,,,,, +14,0.4999999403953552,9.809534072875977,,,,,,,,,,,,,, +15,0.4999999403953552,9.726783752441406,,,,,,,,,,,,,, +16,0.4999998807907104,9.64841079711914,,,,,,,,,,,,,, +17,0.499999850988388,9.544710159301758,,,,,,,,,,,,,, +18,0.4999998211860657,9.484222412109377,,,,,,,,,,,,,, +19,0.499999850988388,9.451597213745115,,,,,,,,,,,,,, +20,0.4999998211860657,9.366528511047363,,,,,,,,,,,,,, +21,0.4999997913837433,9.37139129638672,,,,,,,,,,,,,, +22,0.4999998211860657,9.320984840393066,,,,,,,,,,,,,, +23,0.4999997913837433,9.290326118469238,,,,,,,,,,,,,, +24,0.4999998211860657,9.25490379333496,,,,,,,,,,,,,, +25,0.4999997615814209,9.190329551696776,,,,,,,,,,,,,, +26,0.4999998211860657,9.236665725708008,,,,,,,,,,,,,, +27,0.4999998211860657,9.224600791931152,,,,,,,,,,,,,, +28,0.4999997317790985,9.17792510986328,,,,,,,,,,,,,, +29,0.4999998211860657,9.159496307373049,,,,,,,,,,,,,, +30,0.4999997615814209,9.129840850830078,,,,,,,,,,,,,, +31,0.4999997615814209,9.13117218017578,,,,,,,,,,,,,, +32,0.4999998211860657,9.014959335327148,,,,,,,,,,,,,, +33,0.4999997615814209,9.065778732299805,,,,,,,,,,,,,, +34,0.4999997913837433,9.02350616455078,,,,,,,,,,,,,, +35,0.4999997913837433,8.962442398071289,,,,,,,,,,,,,, +36,0.4999997317790985,8.963725090026855,,,,,,,,,,,,,, +37,0.4999997615814209,8.918502807617188,,,,,,,,,,,,,, +38,0.4999997913837433,8.906380653381348,,,,,,,,,,,,,, +39,0.4999997019767761,8.837135314941406,,,,,,,,,,,,,, +40,0.4999997913837433,8.824031829833984,,,,,,,,,,,,,, +41,0.4999997913837433,8.862309455871582,,,,,,,,,,,,,, +42,0.4999997615814209,8.813311576843262,,,,,,,,,,,,,, +43,0.4999997615814209,8.740362167358398,,,,,,,,,,,,,, +44,0.4999997913837433,8.72372817993164,,,,,,,,,,,,,, +45,0.4999997317790985,8.735066413879395,,,,,,,,,,,,,, +46,0.4999997317790985,8.705228805541992,,,,,,,,,,,,,, +47,0.4999997615814209,8.647663116455078,,,,,,,,,,,,,, +48,0.4999997019767761,8.709701538085938,,,,,,,,,,,,,, +49,0.4999997615814209,8.645195007324219,,,,,,,,,,,,,, +50,0.4999997019767761,8.646834373474121,,,,,,,,,,,,,, +51,0.4999997317790985,8.589305877685547,,,,,,,,,,,,,, +52,0.4999997913837433,8.587969779968262,,,,,,,,,,,,,, +53,0.499999850988388,8.591266632080078,,,,,,,,,,,,,, +54,0.4999997615814209,8.553041458129883,,,,,,,,,,,,,, +55,0.4999997317790985,8.527400016784668,,,,,,,,,,,,,, +56,0.4999997615814209,8.483293533325195,,,,,,,,,,,,,, +57,0.4999997913837433,8.562009811401367,,,,,,,,,,,,,, +58,0.4999997615814209,8.494300842285156,,,,,,,,,,,,,, +59,0.4999997317790985,8.475037574768066,,,,,,,,,,,,,, +60,0.4999997317790985,8.44682502746582,,,,,,,,,,,,,, +61,0.4999997317790985,8.340999603271484,,,,,,,,,,,,,, +62,0.4999997913837433,8.410381317138672,,,,,,,,,,,,,, +63,0.4999997615814209,8.331961631774902,,,,,,,,,,,,,, +64,0.4999996721744537,8.321001052856445,,,,,,,,,,,,,, +65,0.4999997615814209,8.297592163085938,,,,,,,,,,,,,, +66,0.4999997019767761,8.303864479064941,,,,,,,,,,,,,, +67,0.4999997615814209,8.260303497314453,,,,,,,,,,,,,, +68,0.4999998211860657,8.242513656616211,,,,,,,,,,,,,, +69,0.4999997615814209,8.174386978149414,,,,,,,,,,,,,, +70,0.4999998211860657,8.19597339630127,,,,,,,,,,,,,, +71,0.4999998211860657,8.162595748901367,,,,,,,,,,,,,, +72,0.4999997615814209,8.232728004455566,,,,,,,,,,,,,, +73,0.4999997913837433,8.148898124694824,,,,,,,,,,,,,, +74,0.499999850988388,8.093880653381348,,,,,,,,,,,,,, +75,0.4999997317790985,8.163352966308594,,,,,,,,,,,,,, +76,0.4999997615814209,8.088354110717773,,,,,,,,,,,,,, +77,0.4999997317790985,8.04184341430664,,,,,,,,,,,,,, +78,0.4999997615814209,8.061649322509766,,,,,,,,,,,,,, +79,0.4999997913837433,8.03415584564209,,,,,,,,,,,,,, +80,0.4999998211860657,8.003983497619629,,,,,,,,,,,,,, +81,0.4999997317790985,7.945516586303711,,,,,,,,,,,,,, +82,0.499999850988388,7.9341511726379395,,,,,,,,,,,,,, +83,0.4999997019767761,7.951496124267578,,,,,,,,,,,,,, +84,0.4999997913837433,7.911312103271484,,,,,,,,,,,,,, +85,0.4999997615814209,7.967821598052978,,,,,,,,,,,,,, +86,0.4999997913837433,7.883920192718506,,,,,,,,,,,,,, +87,0.4999997019767761,7.878758907318115,,,,,,,,,,,,,, +88,0.4999997317790985,7.797598838806152,,,,,,,,,,,,,, +89,0.4999998807907104,7.785647392272949,,,,,,,,,,,,,, +90,0.4999998211860657,7.753024578094482,,,,,,,,,,,,,, +91,0.4999997615814209,7.841927528381348,,,,,,,,,,,,,, +92,0.4999997615814209,7.805875778198242,,,,,,,,,,,,,, +93,0.4999998211860657,7.748126029968262,,,,,,,,,,,,,, +94,0.4999998807907104,7.793522834777832,,,,,,,,,,,,,, +95,0.4999998211860657,7.7534637451171875,,,,,,,,,,,,,, +96,0.4999997615814209,7.67701244354248,,,,,,,,,,,,,, +97,0.4999997019767761,7.7434000968933105,,,,,,,,,,,,,, +98,0.499999850988388,7.767987251281738,,,,,,,,,,,,,, +99,0.4999997019767761,7.667636871337891,,,,,,,,,,,,,, +100,0.499999850988388,7.612526893615723,,,,,,,,,,,,,, +500,0.4999996423721313,5.629510879516602,,,,,,,,,,,,,, +1000,0.4999992251396179,4.550928115844727,,,,,,,,,,,,,, +1500,0.48031747341156,4.190935134887695,,,,,,,,,,,,,, +2000,0.3120012581348419,3.934011936187744,,,,,,,,,,,,,, +2500,0.300648957490921,3.887841463088989,,,,,,,,,,,,,, +3000,0.2809959948062897,3.681372165679932,,,,,,,,,,,,,, +3500,0.2622063755989074,3.695353746414185,,,,,,,,,,,,,, +4000,0.2329761534929275,3.509620428085327,,,,,,,,,,,,,, +4500,0.2197501510381698,3.5959067344665527,,,,,,,,,,,,,, +5000,0.2102813273668289,3.4236321449279785,,,,,,,,,,,,,, +5500,0.1980506926774978,3.51651382446289,,,,,,,,,,,,,, +6000,0.1878391951322555,3.438794612884521,,,,,,,,,,,,,, +6116,,,3.46959228515625,32.123642488533434,3.461498836117328,31.86470070464744,100000.0,0.0,1.0,0.0,2611.8812248706818,3030.7117323875427,2611.8812248706818,414.7203345298767,0.0275413990020751,0.0 +6500,0.1835102438926696,3.486614942550659,,,,,,,,,,,,,, +7000,0.1759923845529556,3.43591833114624,,,,,,,,,,,,,, +7500,0.1631848216056823,3.382352113723755,,,,,,,,,,,,,, +8000,0.1593898087739944,3.3042540550231934,,,,,,,,,,,,,, +8500,0.1574627459049224,3.4077067375183105,,,,,,,,,,,,,, +9000,0.1530763059854507,3.2895798683166504,,,,,,,,,,,,,, +9500,0.145874485373497,3.277822256088257,,,,,,,,,,,,,, +10000,0.1376629620790481,3.283009052276612,,,,,,,,,,,,,, +10500,0.139400228857994,3.284064531326294,,,,,,,,,,,,,, +11000,0.1363649517297744,3.356990337371826,,,,,,,,,,,,,, +11500,0.1325877755880355,3.340426445007324,,,,,,,,,,,,,, +12000,0.1291247308254242,3.2622222900390625,,,,,,,,,,,,,, +12235,,,3.29146728515625,26.88227870077577,3.280596876998082,26.591639902217743,100000.0,0.0,1.0,0.0,5180.229433774948,5790.969086408615,5180.229433774948,603.395170211792,0.0573127269744873,0.0 +12500,0.125816062092781,3.199220895767212,,,,,,,,,,,,,, +13000,0.120329663157463,3.2214035987854004,,,,,,,,,,,,,, +13500,0.1193437278270721,3.177230834960937,,,,,,,,,,,,,, +14000,0.1182410269975662,3.310978412628174,,,,,,,,,,,,,, +14500,0.1172355636954307,3.2833731174468994,,,,,,,,,,,,,, +15000,0.1132587119936943,3.149517774581909,,,,,,,,,,,,,, +15500,0.109029434621334,3.101914167404175,,,,,,,,,,,,,, +16000,0.1098910495638847,3.2445619106292725,,,,,,,,,,,,,, +16500,0.1064804196357727,3.1635241508483887,,,,,,,,,,,,,, +17000,0.1038777530193328,3.2243142127990723,,,,,,,,,,,,,, +17500,0.1060129553079605,3.120731830596924,,,,,,,,,,,,,, +18000,0.1020874679088592,3.225571155548096,,,,,,,,,,,,,, +18363,,,3.2044586181640624,24.642155589132592,3.1929507048233696,24.360201262704976,100000.0,0.0,1.0,0.0,7748.8618953228,8551.51058459282,7748.8618953228,792.2495367527008,0.0849885940551757,0.0 +18500,0.1039405241608619,3.1287682056427,,,,,,,,,,,,,, +19000,0.1089193299412727,3.186685085296631,,,,,,,,,,,,,, +19500,0.0981461703777313,3.151864051818848,,,,,,,,,,,,,, +20000,0.1018523573875427,3.0459647178649902,,,,,,,,,,,,,, +20500,0.0998804196715354,3.1840708255767822,,,,,,,,,,,,,, +21000,0.098125122487545,3.165796995162964,,,,,,,,,,,,,, +21500,0.0915616005659103,3.093239545822144,,,,,,,,,,,,,, +22000,0.0948603004217147,3.169605255126953,,,,,,,,,,,,,, +22500,0.091604933142662,3.1871609687805176,,,,,,,,,,,,,, +23000,0.0921157598495483,3.241499900817871,,,,,,,,,,,,,, +23500,0.0993161052465438,3.2438671588897705,,,,,,,,,,,,,, +24000,0.0904274210333824,3.0954127311706543,,,,,,,,,,,,,, +24490,,,3.145959091186524,23.24195594217749,3.134155585637788,22.969232101313764,100000.0,0.0,1.0,0.0,10317.61235523224,11311.779911279678,10317.61235523224,980.7469322681428,0.1141092777252197,0.0 +24500,0.0911130681633949,3.148308038711548,,,,,,,,,,,,,, +25000,0.0873359888792038,3.1186017990112305,,,,,,,,,,,,,, +25500,0.0872185751795768,3.059952974319458,,,,,,,,,,,,,, +26000,0.0964387878775596,3.0447826385498047,,,,,,,,,,,,,, +26500,0.0871369838714599,3.0272116661071777,,,,,,,,,,,,,, +27000,0.083136461675167,2.997666120529175,,,,,,,,,,,,,, +27500,0.085715465247631,3.136771440505981,,,,,,,,,,,,,, +28000,0.093138538300991,3.142975330352783,,,,,,,,,,,,,, +28500,0.0897620171308517,3.151700019836426,,,,,,,,,,,,,, +29000,0.0879644080996513,3.069101095199585,,,,,,,,,,,,,, +29500,0.0833689868450164,3.044318199157715,,,,,,,,,,,,,, +30000,0.0854799449443817,3.023466348648072,,,,,,,,,,,,,, +30500,0.0846617594361305,3.1122360229492188,,,,,,,,,,,,,, +30619,,,3.099249076843262,22.181288582778382,3.0879880314897696,21.932905238908088,100000.0,0.0,1.0,0.0,12886.524369478226,14072.697648763657,12886.524369478226,1169.740087032318,0.1460838317871093,0.0 +30619,,,,,,,,,,,12886.524369478226,,,,,0.0 diff --git a/logs/self_tuning/ademamix_golden/study_2/finewebedu_lm_pytorch/trial_1/meta_data_0.json b/logs/self_tuning/ademamix_golden/study_2/finewebedu_lm_pytorch/trial_1/meta_data_0.json new file mode 100644 index 00000000..f5e37236 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/finewebedu_lm_pytorch/trial_1/meta_data_0.json @@ -0,0 +1,69 @@ +{ + "workload.activation": "silu", + "workload.attention_temp": 1.0, + "workload.eval_batch_size": 256, + "workload.eval_period_time_sec": 2571, + "workload.glu": true, + "workload.max_allowed_runtime_sec": 31967, + "workload.num_eval_train_examples": 10000, + "workload.num_test_examples": 0, + "workload.num_train_examples": 8749870, + "workload.num_validation_examples": 100000, + "workload.pre_ln": true, + "workload.step_hint": 72000, + "workload.target_metric_name": "ppl", + "workload.validation_target_value": 22.2995, + "workload.warmup_factor": 0.1, + "cpu.util.avg_percent_since_last": 16.2, + "cpu.freq.current": 2200.1639999999993, + "mem.total": 359053524992, + "mem.available": 349594542080, + "mem.used": 6303170560, + "mem.percent_used": 2.6, + "mem.read_bytes_since_boot": 17952947200, + "mem.write_bytes_since_boot": 37424890880, + "net.bytes_sent_since_boot": 31246124, + "net.bytes_recv_since_boot": 31249086, + "gpu.count": 4, + "gpu.0.compute.util": 0.0, + "gpu.0.mem.util": 0.0917724609375, + "gpu.0.mem.total": 40960.0, + "gpu.0.mem.used": 3759.0, + "gpu.0.mem.free": 36568.0, + "gpu.0.temp.current": 40.0, + "gpu.1.compute.util": 0.0, + "gpu.1.mem.util": 0.0917724609375, + "gpu.1.mem.total": 40960.0, + "gpu.1.mem.used": 3759.0, + "gpu.1.mem.free": 36568.0, + "gpu.1.temp.current": 48.0, + "gpu.2.compute.util": 0.0, + "gpu.2.mem.util": 0.0917724609375, + "gpu.2.mem.total": 40960.0, + "gpu.2.mem.used": 3759.0, + "gpu.2.mem.free": 36568.0, + "gpu.2.temp.current": 40.0, + "gpu.3.compute.util": 0.0, + "gpu.3.mem.util": 0.0917724609375, + "gpu.3.mem.total": 40960.0, + "gpu.3.mem.used": 3759.0, + "gpu.3.mem.free": 36568.0, + "gpu.3.temp.current": 47.0, + "gpu.avg.compute.util": 0.0, + "gpu.avg.mem.util": 0.0917724609375, + "gpu.avg.mem.total": 40960.0, + "gpu.avg.mem.used": 3759.0, + "gpu.avg.mem.free": 36568.0, + "gpu.avg.temp.current": 43.75, + "os_platform": "Linux-6.1.0-44-cloud-amd64-x86_64-with-glibc2.31", + "python_version": "3.11.10", + "python_compiler": "GCC 9.4.0", + "git_branch": "main", + "git_commit_hash": "b21be29be0a1573fb4f78f849aea019cdb520862", + "cpu_model_name": "Intel(R) Xeon(R) CPU @ 2.20GHz", + "cpu_count": 24, + "gpu_model_name": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_driver": "550.90.12", + "rng_seed": -965294040 +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/imagenet_resnet_pytorch_09-13-2026-02-40-09.log b/logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/imagenet_resnet_pytorch_09-13-2026-02-40-09.log new file mode 100644 index 00000000..3031dbfb --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/imagenet_resnet_pytorch_09-13-2026-02-40-09.log @@ -0,0 +1,1027 @@ +torchrun --redirects 1:0,2:0,3:0 --standalone --nnodes=1 --nproc_per_node=4 submission_runner.py --framework=pytorch --workload=imagenet_resnet --submission_path=submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py --data_dir=/data/imagenet/pytorch --experiment_dir=/experiment_runs --experiment_name=submissions_a100/ademamix_golden/study_2 --overwrite=True --save_checkpoints=False --rng_seed=-1587548617 --imagenet_v2_data_dir=/data/imagenet/pytorch --torch_compile=true --tuning_ruleset=self 2>&1 | tee -a /logs/imagenet_resnet_pytorch_09-13-2026-02-40-09.log +W0913 02:40:10.784000 9 site-packages/torch/distributed/run.py:803] +W0913 02:40:10.784000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0913 02:40:10.784000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0913 02:40:10.784000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-13 02:40:12.413823: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-13 02:40:12.413820: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-13 02:40:12.413851: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-13 02:40:12.413852: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789267212.437699 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789267212.437698 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789267212.437699 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789267212.437777 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789267212.445367 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789267212.445390 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789267212.445391 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789267212.445398 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789267212.468056 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789267212.468060 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789267212.468057 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789267212.468053 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789267212.468085 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789267212.468085 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789267212.468085 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789267212.468087 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789267212.468088 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789267212.468088 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789267212.468090 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789267212.468090 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789267212.468090 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789267212.468091 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789267212.468093 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789267212.468095 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789267218.335019 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789267218.488937 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789267218.514174 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789267218.715656 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W913 02:40:23.840357143 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0913 02:40:24.284107 140650416301248 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/imagenet_resnet_pytorch. +I0913 02:40:24.284112 140458808919232 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/imagenet_resnet_pytorch. +I0913 02:40:24.284110 140557890745536 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/imagenet_resnet_pytorch. +I0913 02:40:24.284136 140514271229120 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/imagenet_resnet_pytorch. +I0913 02:40:24.307265 140514271229120 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/imagenet_resnet_pytorch/trial_1. +I0913 02:40:24.644655 140514271229120 submission_runner.py:242] Initializing dataset. +I0913 02:40:43.101076 140514271229120 submission_runner.py:251] Initializing model. +I0913 02:40:43.563641 140514271229120 submission_runner.py:290] Performing `torch.compile`. +I0913 02:40:44.634777 140514271229120 submission_runner.py:294] Initializing optimizer. +I0913 02:40:44.635635 140514271229120 submission_runner.py:299] Initializing metrics bundle. +I0913 02:40:44.635784 140514271229120 submission_runner.py:321] Initializing checkpoint and logger. +I0913 02:40:44.636687 140514271229120 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_2/imagenet_resnet_pytorch/trial_1/meta_data_0.json. +I0913 02:40:44.898468 140514271229120 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_2/imagenet_resnet_pytorch/trial_1/flags_0.json. +I0913 02:40:44.936172 140514271229120 submission_runner.py:359] Starting training loop. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +[rank2]:W0913 02:40:59.315000 40 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank1]:W0913 02:41:00.103000 39 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank0]:W0913 02:41:04.265000 38 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank3]:W0913 02:41:08.226000 41 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +/usr/local/lib/python3.11/site-packages/torch/autograd/graph.py:841: UserWarning: Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 512, 512] +bucket_view.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 1, 1] (Triggered internally at /pytorch/torch/csrc/distributed/c10d/reducer.cpp:334.) + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +/usr/local/lib/python3.11/site-packages/torch/autograd/graph.py:841: UserWarning: Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 512, 512] +bucket_view.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 1, 1] (Triggered internally at /pytorch/torch/csrc/distributed/c10d/reducer.cpp:334.) + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +/usr/local/lib/python3.11/site-packages/torch/autograd/graph.py:841: UserWarning: Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 512, 512] +bucket_view.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 1, 1] (Triggered internally at /pytorch/torch/csrc/distributed/c10d/reducer.cpp:334.) + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +/usr/local/lib/python3.11/site-packages/torch/autograd/graph.py:841: UserWarning: Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 512, 512] +bucket_view.sizes() = [2048, 512, 1, 1], strides() = [512, 1, 1, 1] (Triggered internally at /pytorch/torch/csrc/distributed/c10d/reducer.cpp:334.) + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +I0913 02:42:42.583542 140494889809664 logging_writer.py:48] [0] global_step=0, grad_norm=0.499999, loss=6.92216 +I0913 02:42:42.628801 140514271229120 submission.py:307] 0) loss = 6.922, grad_norm = 0.500 +I0913 02:42:43.306879 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +I0913 02:43:44.957569 140514271229120 spec.py:346] Evaluating on the validation split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 02:45:23.341403 140514271229120 spec.py:363] Evaluating on the test split. +I0913 02:45:23.560228 140514271229120 dataset_info.py:707] Load dataset info from /data/imagenet/pytorch/imagenet_v2/matched-frequency/3.0.0 +I0913 02:45:23.599432 140514271229120 reader.py:262] Creating a tf.data.Dataset reading 16 files located in folders: /data/imagenet/pytorch/imagenet_v2/matched-frequency/3.0.0. +I0913 02:45:23.672708 140514271229120 logging_logger.py:49] Constructing tf.data.Dataset imagenet_v2 for split test, from /data/imagenet/pytorch/imagenet_v2/matched-frequency/3.0.0 +I0913 02:45:51.340041 140514271229120 submission_runner.py:516] Time since start: 306.40s, Step: 1, {'train/accuracy': 0.0010363520408163266, 'train/loss': 6.913094034000319, 'validation/accuracy': 0.00048, 'validation/loss': 6.91549375, 'validation/num_examples': 50000, 'test/accuracy': 0.0011, 'test/loss': 6.913284375, 'test/num_examples': 10000, 'score': 117.69383144378662, 'total_duration': 306.40399718284607, 'accumulated_submission_time': 117.69383144378662, 'accumulated_eval_time': 188.03324580192566, 'accumulated_logging_time': 0} +I0913 02:45:51.361833 140455723267840 logging_writer.py:48] [1] accumulated_eval_time=188.033, accumulated_logging_time=0, accumulated_submission_time=117.694, global_step=1, preemption_count=0, score=117.694, test/accuracy=0.0011, test/loss=6.91328, test/num_examples=10000, total_duration=306.404, train/accuracy=0.00103635, train/loss=6.91309, validation/accuracy=0.00048, validation/loss=6.91549, validation/num_examples=50000 +I0913 02:45:53.121860 140455714875136 logging_writer.py:48] [1] global_step=1, grad_norm=0.499999, loss=6.92548 +I0913 02:45:53.124892 140514271229120 submission.py:307] 1) loss = 6.925, grad_norm = 0.500 +I0913 02:45:53.410806 140455723267840 logging_writer.py:48] [2] global_step=2, grad_norm=0.499999, loss=6.92674 +I0913 02:45:53.414064 140514271229120 submission.py:307] 2) loss = 6.927, grad_norm = 0.500 +I0913 02:45:53.687753 140455714875136 logging_writer.py:48] [3] global_step=3, grad_norm=0.499999, loss=6.92515 +I0913 02:45:53.690816 140514271229120 submission.py:307] 3) loss = 6.925, grad_norm = 0.500 +I0913 02:45:54.029730 140455723267840 logging_writer.py:48] [4] global_step=4, grad_norm=0.499999, loss=6.92283 +I0913 02:45:54.032961 140514271229120 submission.py:307] 4) loss = 6.923, grad_norm = 0.500 +I0913 02:45:54.431356 140455714875136 logging_writer.py:48] [5] global_step=5, grad_norm=0.499999, loss=6.92046 +I0913 02:45:54.434638 140514271229120 submission.py:307] 5) loss = 6.920, grad_norm = 0.500 +I0913 02:45:54.745478 140455723267840 logging_writer.py:48] [6] global_step=6, grad_norm=0.499999, loss=6.91916 +I0913 02:45:54.748706 140514271229120 submission.py:307] 6) loss = 6.919, grad_norm = 0.500 +I0913 02:45:55.075903 140455714875136 logging_writer.py:48] [7] global_step=7, grad_norm=0.499999, loss=6.929 +I0913 02:45:55.079237 140514271229120 submission.py:307] 7) loss = 6.929, grad_norm = 0.500 +I0913 02:45:55.435167 140455723267840 logging_writer.py:48] [8] global_step=8, grad_norm=0.499999, loss=6.91383 +I0913 02:45:55.439304 140514271229120 submission.py:307] 8) loss = 6.914, grad_norm = 0.500 +I0913 02:45:55.824584 140455714875136 logging_writer.py:48] [9] global_step=9, grad_norm=0.499999, loss=6.91548 +I0913 02:45:55.829273 140514271229120 submission.py:307] 9) loss = 6.915, grad_norm = 0.500 +I0913 02:45:56.149567 140455723267840 logging_writer.py:48] [10] global_step=10, grad_norm=0.499999, loss=6.93493 +I0913 02:45:56.154302 140514271229120 submission.py:307] 10) loss = 6.935, grad_norm = 0.500 +I0913 02:45:56.456129 140455714875136 logging_writer.py:48] [11] global_step=11, grad_norm=0.499999, loss=6.9334 +I0913 02:45:56.459509 140514271229120 submission.py:307] 11) loss = 6.933, grad_norm = 0.500 +I0913 02:45:56.772262 140455723267840 logging_writer.py:48] [12] global_step=12, grad_norm=0.499999, loss=6.92623 +I0913 02:45:56.777846 140514271229120 submission.py:307] 12) loss = 6.926, grad_norm = 0.500 +I0913 02:45:57.111949 140455714875136 logging_writer.py:48] [13] global_step=13, grad_norm=0.499999, loss=6.92665 +I0913 02:45:57.130725 140514271229120 submission.py:307] 13) loss = 6.927, grad_norm = 0.500 +I0913 02:45:57.457497 140455723267840 logging_writer.py:48] [14] global_step=14, grad_norm=0.499999, loss=6.93356 +I0913 02:45:57.464560 140514271229120 submission.py:307] 14) loss = 6.934, grad_norm = 0.500 +I0913 02:45:57.834955 140455714875136 logging_writer.py:48] [15] global_step=15, grad_norm=0.499999, loss=6.92486 +I0913 02:45:57.840969 140514271229120 submission.py:307] 15) loss = 6.925, grad_norm = 0.500 +I0913 02:45:58.209248 140455723267840 logging_writer.py:48] [16] global_step=16, grad_norm=0.499999, loss=6.93278 +I0913 02:45:58.221342 140514271229120 submission.py:307] 16) loss = 6.933, grad_norm = 0.500 +I0913 02:45:58.558937 140455714875136 logging_writer.py:48] [17] global_step=17, grad_norm=0.499999, loss=6.91763 +I0913 02:45:58.570076 140514271229120 submission.py:307] 17) loss = 6.918, grad_norm = 0.500 +I0913 02:45:58.942667 140455723267840 logging_writer.py:48] [18] global_step=18, grad_norm=0.499999, loss=6.92143 +I0913 02:45:58.962928 140514271229120 submission.py:307] 18) loss = 6.921, grad_norm = 0.500 +I0913 02:45:59.276869 140455714875136 logging_writer.py:48] [19] global_step=19, grad_norm=0.499999, loss=6.92033 +I0913 02:45:59.292801 140514271229120 submission.py:307] 19) loss = 6.920, grad_norm = 0.500 +I0913 02:45:59.609597 140455723267840 logging_writer.py:48] [20] global_step=20, grad_norm=0.499999, loss=6.92592 +I0913 02:45:59.613582 140514271229120 submission.py:307] 20) loss = 6.926, grad_norm = 0.500 +I0913 02:45:59.956305 140455714875136 logging_writer.py:48] [21] global_step=21, grad_norm=0.499999, loss=6.92787 +I0913 02:45:59.959499 140514271229120 submission.py:307] 21) loss = 6.928, grad_norm = 0.500 +I0913 02:46:00.222577 140455723267840 logging_writer.py:48] [22] global_step=22, grad_norm=0.499999, loss=6.925 +I0913 02:46:00.237239 140514271229120 submission.py:307] 22) loss = 6.925, grad_norm = 0.500 +I0913 02:46:00.573362 140455714875136 logging_writer.py:48] [23] global_step=23, grad_norm=0.499999, loss=6.9245 +I0913 02:46:00.607747 140514271229120 submission.py:307] 23) loss = 6.925, grad_norm = 0.500 +I0913 02:46:00.990289 140455723267840 logging_writer.py:48] [24] global_step=24, grad_norm=0.499999, loss=6.91689 +I0913 02:46:01.036317 140514271229120 submission.py:307] 24) loss = 6.917, grad_norm = 0.500 +I0913 02:46:01.456398 140455714875136 logging_writer.py:48] [25] global_step=25, grad_norm=0.499999, loss=6.92513 +I0913 02:46:01.489113 140514271229120 submission.py:307] 25) loss = 6.925, grad_norm = 0.500 +I0913 02:46:01.816308 140455723267840 logging_writer.py:48] [26] global_step=26, grad_norm=0.499999, loss=6.92196 +I0913 02:46:01.821289 140514271229120 submission.py:307] 26) loss = 6.922, grad_norm = 0.500 +I0913 02:46:02.094276 140455714875136 logging_writer.py:48] [27] global_step=27, grad_norm=0.499999, loss=6.91605 +I0913 02:46:02.097264 140514271229120 submission.py:307] 27) loss = 6.916, grad_norm = 0.500 +I0913 02:46:02.386655 140455723267840 logging_writer.py:48] [28] global_step=28, grad_norm=0.499999, loss=6.93263 +I0913 02:46:02.389662 140514271229120 submission.py:307] 28) loss = 6.933, grad_norm = 0.500 +I0913 02:46:02.675414 140455714875136 logging_writer.py:48] [29] global_step=29, grad_norm=0.499999, loss=6.92304 +I0913 02:46:02.679157 140514271229120 submission.py:307] 29) loss = 6.923, grad_norm = 0.500 +I0913 02:46:03.084880 140455723267840 logging_writer.py:48] [30] global_step=30, grad_norm=0.499999, loss=6.92744 +I0913 02:46:03.111455 140514271229120 submission.py:307] 30) loss = 6.927, grad_norm = 0.500 +I0913 02:46:03.557242 140455714875136 logging_writer.py:48] [31] global_step=31, grad_norm=0.499999, loss=6.92882 +I0913 02:46:03.577655 140514271229120 submission.py:307] 31) loss = 6.929, grad_norm = 0.500 +I0913 02:46:03.898907 140455723267840 logging_writer.py:48] [32] global_step=32, grad_norm=0.499999, loss=6.91806 +I0913 02:46:03.902130 140514271229120 submission.py:307] 32) loss = 6.918, grad_norm = 0.500 +I0913 02:46:04.216420 140455714875136 logging_writer.py:48] [33] global_step=33, grad_norm=0.499999, loss=6.91804 +I0913 02:46:04.220784 140514271229120 submission.py:307] 33) loss = 6.918, grad_norm = 0.500 +I0913 02:46:04.514967 140455723267840 logging_writer.py:48] [34] global_step=34, grad_norm=0.499999, loss=6.92402 +I0913 02:46:04.518010 140514271229120 submission.py:307] 34) loss = 6.924, grad_norm = 0.500 +I0913 02:46:04.885226 140455714875136 logging_writer.py:48] [35] global_step=35, grad_norm=0.499999, loss=6.92649 +I0913 02:46:04.935875 140514271229120 submission.py:307] 35) loss = 6.926, grad_norm = 0.500 +I0913 02:46:05.541225 140455723267840 logging_writer.py:48] [36] global_step=36, grad_norm=0.499999, loss=6.92062 +I0913 02:46:05.587137 140514271229120 submission.py:307] 36) loss = 6.921, grad_norm = 0.500 +I0913 02:46:05.995994 140455714875136 logging_writer.py:48] [37] global_step=37, grad_norm=0.499999, loss=6.92242 +I0913 02:46:05.999060 140514271229120 submission.py:307] 37) loss = 6.922, grad_norm = 0.500 +I0913 02:46:06.294274 140455723267840 logging_writer.py:48] [38] global_step=38, grad_norm=0.499999, loss=6.91125 +I0913 02:46:06.297266 140514271229120 submission.py:307] 38) loss = 6.911, grad_norm = 0.500 +I0913 02:46:06.593253 140455714875136 logging_writer.py:48] [39] global_step=39, grad_norm=0.499999, loss=6.92818 +I0913 02:46:06.600039 140514271229120 submission.py:307] 39) loss = 6.928, grad_norm = 0.500 +I0913 02:46:07.130118 140455723267840 logging_writer.py:48] [40] global_step=40, grad_norm=0.499999, loss=6.92285 +I0913 02:46:07.148998 140514271229120 submission.py:307] 40) loss = 6.923, grad_norm = 0.500 +I0913 02:46:07.721217 140455714875136 logging_writer.py:48] [41] global_step=41, grad_norm=0.499999, loss=6.92041 +I0913 02:46:07.755089 140514271229120 submission.py:307] 41) loss = 6.920, grad_norm = 0.500 +I0913 02:46:08.165732 140455723267840 logging_writer.py:48] [42] global_step=42, grad_norm=0.499999, loss=6.91592 +I0913 02:46:08.171030 140514271229120 submission.py:307] 42) loss = 6.916, grad_norm = 0.500 +I0913 02:46:08.491838 140455714875136 logging_writer.py:48] [43] global_step=43, grad_norm=0.499999, loss=6.91588 +I0913 02:46:08.495609 140514271229120 submission.py:307] 43) loss = 6.916, grad_norm = 0.500 +I0913 02:46:08.865222 140455723267840 logging_writer.py:48] [44] global_step=44, grad_norm=0.499999, loss=6.91888 +I0913 02:46:08.875980 140514271229120 submission.py:307] 44) loss = 6.919, grad_norm = 0.500 +I0913 02:46:09.311389 140455714875136 logging_writer.py:48] [45] global_step=45, grad_norm=0.499999, loss=6.92483 +I0913 02:46:09.323806 140514271229120 submission.py:307] 45) loss = 6.925, grad_norm = 0.500 +I0913 02:46:09.845654 140455723267840 logging_writer.py:48] [46] global_step=46, grad_norm=0.499999, loss=6.91691 +I0913 02:46:09.851060 140514271229120 submission.py:307] 46) loss = 6.917, grad_norm = 0.500 +I0913 02:46:10.159313 140455714875136 logging_writer.py:48] [47] global_step=47, grad_norm=0.499999, loss=6.92445 +I0913 02:46:10.162402 140514271229120 submission.py:307] 47) loss = 6.924, grad_norm = 0.500 +I0913 02:46:10.505329 140455723267840 logging_writer.py:48] [48] global_step=48, grad_norm=0.499999, loss=6.91306 +I0913 02:46:10.508986 140514271229120 submission.py:307] 48) loss = 6.913, grad_norm = 0.500 +I0913 02:46:11.085234 140455714875136 logging_writer.py:48] [49] global_step=49, grad_norm=0.499999, loss=6.91409 +I0913 02:46:11.133007 140514271229120 submission.py:307] 49) loss = 6.914, grad_norm = 0.500 +I0913 02:46:11.816141 140455723267840 logging_writer.py:48] [50] global_step=50, grad_norm=0.499999, loss=6.92213 +I0913 02:46:11.831662 140514271229120 submission.py:307] 50) loss = 6.922, grad_norm = 0.500 +I0913 02:46:12.182626 140455714875136 logging_writer.py:48] [51] global_step=51, grad_norm=0.499999, loss=6.91064 +I0913 02:46:12.186053 140514271229120 submission.py:307] 51) loss = 6.911, grad_norm = 0.500 +I0913 02:46:12.496457 140455723267840 logging_writer.py:48] [52] global_step=52, grad_norm=0.499999, loss=6.91282 +I0913 02:46:12.499747 140514271229120 submission.py:307] 52) loss = 6.913, grad_norm = 0.500 +I0913 02:46:12.817211 140455714875136 logging_writer.py:48] [53] global_step=53, grad_norm=0.499999, loss=6.91586 +I0913 02:46:12.844282 140514271229120 submission.py:307] 53) loss = 6.916, grad_norm = 0.500 +I0913 02:46:13.253211 140455723267840 logging_writer.py:48] [54] global_step=54, grad_norm=0.499999, loss=6.90244 +I0913 02:46:13.308851 140514271229120 submission.py:307] 54) loss = 6.902, grad_norm = 0.500 +I0913 02:46:13.932094 140455714875136 logging_writer.py:48] [55] global_step=55, grad_norm=0.499999, loss=6.90639 +I0913 02:46:14.010385 140514271229120 submission.py:307] 55) loss = 6.906, grad_norm = 0.500 +I0913 02:46:14.336534 140455723267840 logging_writer.py:48] [56] global_step=56, grad_norm=0.499999, loss=6.92391 +I0913 02:46:14.339511 140514271229120 submission.py:307] 56) loss = 6.924, grad_norm = 0.500 +I0913 02:46:14.638348 140455714875136 logging_writer.py:48] [57] global_step=57, grad_norm=0.499999, loss=6.92037 +I0913 02:46:14.643428 140514271229120 submission.py:307] 57) loss = 6.920, grad_norm = 0.500 +I0913 02:46:14.993362 140455723267840 logging_writer.py:48] [58] global_step=58, grad_norm=0.499999, loss=6.90298 +I0913 02:46:15.054990 140514271229120 submission.py:307] 58) loss = 6.903, grad_norm = 0.500 +I0913 02:46:15.604407 140455714875136 logging_writer.py:48] [59] global_step=59, grad_norm=0.499999, loss=6.90966 +I0913 02:46:15.656446 140514271229120 submission.py:307] 59) loss = 6.910, grad_norm = 0.500 +I0913 02:46:16.097481 140455723267840 logging_writer.py:48] [60] global_step=60, grad_norm=0.499999, loss=6.92318 +I0913 02:46:16.102180 140514271229120 submission.py:307] 60) loss = 6.923, grad_norm = 0.500 +I0913 02:46:16.433702 140455714875136 logging_writer.py:48] [61] global_step=61, grad_norm=0.499999, loss=6.9232 +I0913 02:46:16.440587 140514271229120 submission.py:307] 61) loss = 6.923, grad_norm = 0.500 +I0913 02:46:16.736635 140455723267840 logging_writer.py:48] [62] global_step=62, grad_norm=0.499999, loss=6.91336 +I0913 02:46:16.741057 140514271229120 submission.py:307] 62) loss = 6.913, grad_norm = 0.500 +I0913 02:46:17.261626 140455714875136 logging_writer.py:48] [63] global_step=63, grad_norm=0.499999, loss=6.91845 +I0913 02:46:17.285759 140514271229120 submission.py:307] 63) loss = 6.918, grad_norm = 0.500 +I0913 02:46:18.053771 140455723267840 logging_writer.py:48] [64] global_step=64, grad_norm=0.499999, loss=6.91623 +I0913 02:46:18.088458 140514271229120 submission.py:307] 64) loss = 6.916, grad_norm = 0.500 +I0913 02:46:18.484918 140455714875136 logging_writer.py:48] [65] global_step=65, grad_norm=0.499999, loss=6.92355 +I0913 02:46:18.487896 140514271229120 submission.py:307] 65) loss = 6.924, grad_norm = 0.500 +I0913 02:46:18.821224 140455723267840 logging_writer.py:48] [66] global_step=66, grad_norm=0.499999, loss=6.91266 +I0913 02:46:18.840023 140514271229120 submission.py:307] 66) loss = 6.913, grad_norm = 0.500 +I0913 02:46:19.500155 140455714875136 logging_writer.py:48] [67] global_step=67, grad_norm=0.499999, loss=6.90992 +I0913 02:46:19.544495 140514271229120 submission.py:307] 67) loss = 6.910, grad_norm = 0.500 +I0913 02:46:20.146606 140455723267840 logging_writer.py:48] [68] global_step=68, grad_norm=0.499999, loss=6.90758 +I0913 02:46:20.149811 140514271229120 submission.py:307] 68) loss = 6.908, grad_norm = 0.500 +I0913 02:46:20.464291 140455714875136 logging_writer.py:48] [69] global_step=69, grad_norm=0.499999, loss=6.90592 +I0913 02:46:20.480184 140514271229120 submission.py:307] 69) loss = 6.906, grad_norm = 0.500 +I0913 02:46:20.977794 140455723267840 logging_writer.py:48] [70] global_step=70, grad_norm=0.499999, loss=6.91432 +I0913 02:46:21.006054 140514271229120 submission.py:307] 70) loss = 6.914, grad_norm = 0.500 +I0913 02:46:21.507154 140455714875136 logging_writer.py:48] [71] global_step=71, grad_norm=0.499999, loss=6.90337 +I0913 02:46:21.537403 140514271229120 submission.py:307] 71) loss = 6.903, grad_norm = 0.500 +I0913 02:46:22.207471 140455723267840 logging_writer.py:48] [72] global_step=72, grad_norm=0.499999, loss=6.90867 +I0913 02:46:22.288594 140514271229120 submission.py:307] 72) loss = 6.909, grad_norm = 0.500 +I0913 02:46:22.685683 140455714875136 logging_writer.py:48] [73] global_step=73, grad_norm=0.499999, loss=6.91152 +I0913 02:46:22.688845 140514271229120 submission.py:307] 73) loss = 6.912, grad_norm = 0.500 +I0913 02:46:22.985328 140455723267840 logging_writer.py:48] [74] global_step=74, grad_norm=0.499999, loss=6.9133 +I0913 02:46:23.006158 140514271229120 submission.py:307] 74) loss = 6.913, grad_norm = 0.500 +I0913 02:46:23.381552 140455714875136 logging_writer.py:48] [75] global_step=75, grad_norm=0.499999, loss=6.88865 +I0913 02:46:23.407792 140514271229120 submission.py:307] 75) loss = 6.889, grad_norm = 0.500 +I0913 02:46:23.857216 140455723267840 logging_writer.py:48] [76] global_step=76, grad_norm=0.499999, loss=6.91268 +I0913 02:46:23.892663 140514271229120 submission.py:307] 76) loss = 6.913, grad_norm = 0.500 +I0913 02:46:24.305084 140455714875136 logging_writer.py:48] [77] global_step=77, grad_norm=0.499999, loss=6.90356 +I0913 02:46:24.308304 140514271229120 submission.py:307] 77) loss = 6.904, grad_norm = 0.500 +I0913 02:46:24.633614 140455723267840 logging_writer.py:48] [78] global_step=78, grad_norm=0.499999, loss=6.89867 +I0913 02:46:24.665892 140514271229120 submission.py:307] 78) loss = 6.899, grad_norm = 0.500 +I0913 02:46:25.197233 140455714875136 logging_writer.py:48] [79] global_step=79, grad_norm=0.499999, loss=6.90705 +I0913 02:46:25.238072 140514271229120 submission.py:307] 79) loss = 6.907, grad_norm = 0.500 +I0913 02:46:25.772530 140455723267840 logging_writer.py:48] [80] global_step=80, grad_norm=0.499999, loss=6.90579 +I0913 02:46:25.777929 140514271229120 submission.py:307] 80) loss = 6.906, grad_norm = 0.500 +I0913 02:46:26.088659 140455714875136 logging_writer.py:48] [81] global_step=81, grad_norm=0.499999, loss=6.90409 +I0913 02:46:26.091726 140514271229120 submission.py:307] 81) loss = 6.904, grad_norm = 0.500 +I0913 02:46:26.371249 140455723267840 logging_writer.py:48] [82] global_step=82, grad_norm=0.499999, loss=6.89637 +I0913 02:46:26.374248 140514271229120 submission.py:307] 82) loss = 6.896, grad_norm = 0.500 +I0913 02:46:26.733212 140455714875136 logging_writer.py:48] [83] global_step=83, grad_norm=0.499999, loss=6.90591 +I0913 02:46:26.761585 140514271229120 submission.py:307] 83) loss = 6.906, grad_norm = 0.500 +I0913 02:46:27.268262 140455723267840 logging_writer.py:48] [84] global_step=84, grad_norm=0.499999, loss=6.89787 +I0913 02:46:27.288372 140514271229120 submission.py:307] 84) loss = 6.898, grad_norm = 0.500 +I0913 02:46:27.777243 140455714875136 logging_writer.py:48] [85] global_step=85, grad_norm=0.499999, loss=6.89504 +I0913 02:46:27.816313 140514271229120 submission.py:307] 85) loss = 6.895, grad_norm = 0.500 +I0913 02:46:28.123904 140455723267840 logging_writer.py:48] [86] global_step=86, grad_norm=0.499999, loss=6.89378 +I0913 02:46:28.126930 140514271229120 submission.py:307] 86) loss = 6.894, grad_norm = 0.500 +I0913 02:46:28.399043 140455714875136 logging_writer.py:48] [87] global_step=87, grad_norm=0.499999, loss=6.9038 +I0913 02:46:28.403331 140514271229120 submission.py:307] 87) loss = 6.904, grad_norm = 0.500 +I0913 02:46:28.705207 140455723267840 logging_writer.py:48] [88] global_step=88, grad_norm=0.499999, loss=6.89328 +I0913 02:46:28.720600 140514271229120 submission.py:307] 88) loss = 6.893, grad_norm = 0.500 +I0913 02:46:29.197217 140455714875136 logging_writer.py:48] [89] global_step=89, grad_norm=0.499999, loss=6.90544 +I0913 02:46:29.269737 140514271229120 submission.py:307] 89) loss = 6.905, grad_norm = 0.500 +I0913 02:46:29.846504 140455723267840 logging_writer.py:48] [90] global_step=90, grad_norm=0.499999, loss=6.90191 +I0913 02:46:29.853908 140514271229120 submission.py:307] 90) loss = 6.902, grad_norm = 0.500 +I0913 02:46:30.125110 140455714875136 logging_writer.py:48] [91] global_step=91, grad_norm=0.499999, loss=6.89853 +I0913 02:46:30.128157 140514271229120 submission.py:307] 91) loss = 6.899, grad_norm = 0.500 +I0913 02:46:30.435811 140455723267840 logging_writer.py:48] [92] global_step=92, grad_norm=0.499999, loss=6.90409 +I0913 02:46:30.438640 140514271229120 submission.py:307] 92) loss = 6.904, grad_norm = 0.500 +I0913 02:46:30.757209 140455714875136 logging_writer.py:48] [93] global_step=93, grad_norm=0.499999, loss=6.89762 +I0913 02:46:30.776295 140514271229120 submission.py:307] 93) loss = 6.898, grad_norm = 0.500 +I0913 02:46:31.259828 140455723267840 logging_writer.py:48] [94] global_step=94, grad_norm=0.499999, loss=6.89462 +I0913 02:46:31.277759 140514271229120 submission.py:307] 94) loss = 6.895, grad_norm = 0.500 +I0913 02:46:31.733196 140455714875136 logging_writer.py:48] [95] global_step=95, grad_norm=0.499999, loss=6.89926 +I0913 02:46:31.757104 140514271229120 submission.py:307] 95) loss = 6.899, grad_norm = 0.500 +I0913 02:46:32.088593 140455723267840 logging_writer.py:48] [96] global_step=96, grad_norm=0.499999, loss=6.90459 +I0913 02:46:32.092005 140514271229120 submission.py:307] 96) loss = 6.905, grad_norm = 0.500 +I0913 02:46:32.351781 140455714875136 logging_writer.py:48] [97] global_step=97, grad_norm=0.499999, loss=6.89606 +I0913 02:46:32.354762 140514271229120 submission.py:307] 97) loss = 6.896, grad_norm = 0.500 +I0913 02:46:32.614517 140455723267840 logging_writer.py:48] [98] global_step=98, grad_norm=0.499999, loss=6.89268 +I0913 02:46:32.617499 140514271229120 submission.py:307] 98) loss = 6.893, grad_norm = 0.500 +I0913 02:46:32.937855 140455714875136 logging_writer.py:48] [99] global_step=99, grad_norm=0.499999, loss=6.90216 +I0913 02:46:32.948037 140514271229120 submission.py:307] 99) loss = 6.902, grad_norm = 0.500 +I0913 02:46:33.363942 140455723267840 logging_writer.py:48] [100] global_step=100, grad_norm=0.499999, loss=6.89375 +I0913 02:46:33.402614 140514271229120 submission.py:307] 100) loss = 6.894, grad_norm = 0.500 +I0913 02:49:50.761147 140455714875136 logging_writer.py:48] [500] global_step=500, grad_norm=0.499999, loss=6.45762 +I0913 02:49:50.765337 140514271229120 submission.py:307] 500) loss = 6.458, grad_norm = 0.500 +I0913 02:55:23.174470 140455723267840 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.5, loss=5.85945 +I0913 02:55:23.178612 140514271229120 submission.py:307] 1000) loss = 5.859, grad_norm = 0.500 +I0913 03:00:16.838948 140455714875136 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.5, loss=5.41729 +I0913 03:00:16.844303 140514271229120 submission.py:307] 1500) loss = 5.417, grad_norm = 0.500 +I0913 03:05:10.271483 140455723267840 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.5, loss=5.10245 +I0913 03:05:10.275540 140514271229120 submission.py:307] 2000) loss = 5.102, grad_norm = 0.500 +I0913 03:12:55.274756 140455714875136 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.5, loss=4.76279 +I0913 03:12:55.278380 140514271229120 submission.py:307] 2500) loss = 4.763, grad_norm = 0.500 +I0913 03:16:37.112042 140455723267840 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.5, loss=4.36397 +I0913 03:16:37.116330 140514271229120 submission.py:307] 3000) loss = 4.364, grad_norm = 0.500 +I0913 03:19:08.563258 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 03:19:52.705940 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 03:21:07.049565 140514271229120 spec.py:363] Evaluating on the test split. +I0913 03:21:09.604593 140514271229120 submission_runner.py:516] Time since start: 2424.67s, Step: 3229, {'train/accuracy': 0.16212930484693877, 'train/loss': 5.140706276407047, 'validation/accuracy': 0.15864, 'validation/loss': 4.9367896875, 'validation/num_examples': 50000, 'test/accuracy': 0.1165, 'test/loss': 4.81738046875, 'test/num_examples': 10000, 'score': 2108.175397634506, 'total_duration': 2424.668477535248, 'accumulated_submission_time': 2108.175397634506, 'accumulated_eval_time': 309.07459807395935, 'accumulated_logging_time': 0.031135082244873047} +I0913 03:21:09.940368 140456016893696 logging_writer.py:48] [3229] accumulated_eval_time=309.075, accumulated_logging_time=0.0311351, accumulated_submission_time=2108.18, global_step=3229, preemption_count=0, score=2108.18, test/accuracy=0.1165, test/loss=4.81738, test/num_examples=10000, total_duration=2424.67, train/accuracy=0.162129, train/loss=5.14071, validation/accuracy=0.15864, validation/loss=4.93679, validation/num_examples=50000 +I0913 03:24:47.673080 140456025286400 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.5, loss=4.15734 +I0913 03:24:47.676975 140514271229120 submission.py:307] 3500) loss = 4.157, grad_norm = 0.500 +I0913 03:31:09.345628 140456016893696 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.5, loss=3.8814 +I0913 03:31:09.349660 140514271229120 submission.py:307] 4000) loss = 3.881, grad_norm = 0.500 +I0913 03:36:07.930160 140456025286400 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.5, loss=3.5198 +I0913 03:36:07.934064 140514271229120 submission.py:307] 4500) loss = 3.520, grad_norm = 0.500 +I0913 03:44:08.947674 140456016893696 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.5, loss=3.47729 +I0913 03:44:08.951474 140514271229120 submission.py:307] 5000) loss = 3.477, grad_norm = 0.500 +I0913 03:47:59.219186 140456025286400 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.5, loss=3.36777 +I0913 03:47:59.223130 140514271229120 submission.py:307] 5500) loss = 3.368, grad_norm = 0.500 +I0913 03:54:30.879359 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 03:55:23.823305 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 03:56:48.474063 140514271229120 spec.py:363] Evaluating on the test split. +I0913 03:56:49.374275 140514271229120 submission_runner.py:516] Time since start: 4564.44s, Step: 5985, {'train/accuracy': 0.31556919642857145, 'train/loss': 4.7227113684829405, 'validation/accuracy': 0.3118, 'validation/loss': 4.254304375, 'validation/num_examples': 50000, 'test/accuracy': 0.2437, 'test/loss': 3.977594921875, 'test/num_examples': 10000, 'score': 4105.261125326157, 'total_duration': 4564.438182592392, 'accumulated_submission_time': 4105.261125326157, 'accumulated_eval_time': 447.56960701942444, 'accumulated_logging_time': 0.3852415084838867} +I0913 03:56:49.748439 140490695440128 logging_writer.py:48] [5985] accumulated_eval_time=447.57, accumulated_logging_time=0.385242, accumulated_submission_time=4105.26, global_step=5985, preemption_count=0, score=4105.26, test/accuracy=0.2437, test/loss=3.97759, test/num_examples=10000, total_duration=4564.44, train/accuracy=0.315569, train/loss=4.72271, validation/accuracy=0.3118, validation/loss=4.2543, validation/num_examples=50000 +I0913 03:56:54.556773 140487004518144 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.5, loss=3.06484 +I0913 03:56:54.560075 140514271229120 submission.py:307] 6000) loss = 3.065, grad_norm = 0.500 +I0913 04:02:30.662149 140490695440128 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.5, loss=2.82855 +I0913 04:02:30.666058 140514271229120 submission.py:307] 6500) loss = 2.829, grad_norm = 0.500 +I0913 04:08:11.031218 140487004518144 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.5, loss=2.80657 +I0913 04:08:11.035339 140514271229120 submission.py:307] 7000) loss = 2.807, grad_norm = 0.500 +I0913 04:16:08.877966 140490695440128 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.5, loss=2.76293 +I0913 04:16:08.881683 140514271229120 submission.py:307] 7500) loss = 2.763, grad_norm = 0.500 +I0913 04:20:08.286060 140487004518144 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.5, loss=2.56992 +I0913 04:20:08.290126 140514271229120 submission.py:307] 8000) loss = 2.570, grad_norm = 0.500 +I0913 04:27:07.113206 140490695440128 logging_writer.py:48] [8500] global_step=8500, grad_norm=0.5, loss=2.42631 +I0913 04:27:07.118649 140514271229120 submission.py:307] 8500) loss = 2.426, grad_norm = 0.500 +I0913 04:30:11.323523 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 04:30:57.261175 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 04:32:31.163869 140514271229120 spec.py:363] Evaluating on the test split. +I0913 04:32:32.068289 140514271229120 submission_runner.py:516] Time since start: 6707.13s, Step: 8666, {'train/accuracy': 0.4215760522959184, 'train/loss': 4.734850513691804, 'validation/accuracy': 0.4368, 'validation/loss': 3.561740625, 'validation/num_examples': 50000, 'test/accuracy': 0.3598, 'test/loss': 3.02988828125, 'test/num_examples': 10000, 'score': 6103.339204072952, 'total_duration': 6707.132169008255, 'accumulated_submission_time': 6103.339204072952, 'accumulated_eval_time': 588.3147366046906, 'accumulated_logging_time': 0.769068717956543} +I0913 04:32:32.386273 140490695440128 logging_writer.py:48] [8666] accumulated_eval_time=588.315, accumulated_logging_time=0.769069, accumulated_submission_time=6103.34, global_step=8666, preemption_count=0, score=6103.34, test/accuracy=0.3598, test/loss=3.02989, test/num_examples=10000, total_duration=6707.13, train/accuracy=0.421576, train/loss=4.73485, validation/accuracy=0.4368, validation/loss=3.56174, validation/num_examples=50000 +I0913 04:35:29.330907 140490804545280 logging_writer.py:48] [9000] global_step=9000, grad_norm=0.5, loss=2.35037 +I0913 04:35:29.335399 140514271229120 submission.py:307] 9000) loss = 2.350, grad_norm = 0.500 +I0913 04:41:13.698843 140490695440128 logging_writer.py:48] [9500] global_step=9500, grad_norm=0.5, loss=2.404 +I0913 04:41:13.702752 140514271229120 submission.py:307] 9500) loss = 2.404, grad_norm = 0.500 +I0913 04:49:33.522414 140490804545280 logging_writer.py:48] [10000] global_step=10000, grad_norm=0.5, loss=2.13254 +I0913 04:49:33.526086 140514271229120 submission.py:307] 10000) loss = 2.133, grad_norm = 0.500 +I0913 04:53:28.817308 140490695440128 logging_writer.py:48] [10500] global_step=10500, grad_norm=0.5, loss=2.21336 +I0913 04:53:28.821505 140514271229120 submission.py:307] 10500) loss = 2.213, grad_norm = 0.500 +I0913 05:00:39.649753 140490804545280 logging_writer.py:48] [11000] global_step=11000, grad_norm=0.5, loss=2.04634 +I0913 05:00:39.653745 140514271229120 submission.py:307] 11000) loss = 2.046, grad_norm = 0.500 +I0913 05:05:49.948746 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 05:06:42.480536 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 05:07:46.921046 140514271229120 spec.py:363] Evaluating on the test split. +I0913 05:07:47.842917 140514271229120 submission_runner.py:516] Time since start: 8822.91s, Step: 11342, {'train/accuracy': 0.519172512755102, 'train/loss': 3.8327312858737246, 'validation/accuracy': 0.49202, 'validation/loss': 3.432515625, 'validation/num_examples': 50000, 'test/accuracy': 0.419, 'test/loss': 2.7275841796875, 'test/num_examples': 10000, 'score': 8096.4368398189545, 'total_duration': 8822.90674829483, 'accumulated_submission_time': 8096.4368398189545, 'accumulated_eval_time': 706.2089529037476, 'accumulated_logging_time': 1.0964832305908203} +I0913 05:07:48.162351 140490737403648 logging_writer.py:48] [11342] accumulated_eval_time=706.209, accumulated_logging_time=1.09648, accumulated_submission_time=8096.44, global_step=11342, preemption_count=0, score=8096.44, test/accuracy=0.419, test/loss=2.72758, test/num_examples=10000, total_duration=8822.91, train/accuracy=0.519173, train/loss=3.83273, validation/accuracy=0.49202, validation/loss=3.43252, validation/num_examples=50000 +I0913 05:08:50.714383 140490796152576 logging_writer.py:48] [11500] global_step=11500, grad_norm=0.5, loss=1.92423 +I0913 05:08:50.719199 140514271229120 submission.py:307] 11500) loss = 1.924, grad_norm = 0.500 +I0913 05:14:56.493585 140490737403648 logging_writer.py:48] [12000] global_step=12000, grad_norm=0.5, loss=1.94836 +I0913 05:14:56.497921 140514271229120 submission.py:307] 12000) loss = 1.948, grad_norm = 0.500 +I0913 05:23:35.368865 140490796152576 logging_writer.py:48] [12500] global_step=12500, grad_norm=0.5, loss=2.05367 +I0913 05:23:35.372750 140514271229120 submission.py:307] 12500) loss = 2.054, grad_norm = 0.500 +I0913 05:27:34.607588 140490737403648 logging_writer.py:48] [13000] global_step=13000, grad_norm=0.5, loss=1.93117 +I0913 05:27:34.611818 140514271229120 submission.py:307] 13000) loss = 1.931, grad_norm = 0.500 +I0913 05:34:48.692255 140490796152576 logging_writer.py:48] [13500] global_step=13500, grad_norm=0.5, loss=1.89082 +I0913 05:34:48.696388 140514271229120 submission.py:307] 13500) loss = 1.891, grad_norm = 0.500 +I0913 05:41:01.105538 140490737403648 logging_writer.py:48] [14000] global_step=14000, grad_norm=0.5, loss=1.85533 +I0913 05:41:01.109951 140514271229120 submission.py:307] 14000) loss = 1.855, grad_norm = 0.500 +I0913 05:41:05.285223 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 05:42:03.180365 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 05:43:07.490648 140514271229120 spec.py:363] Evaluating on the test split. +I0913 05:43:08.539198 140514271229120 submission_runner.py:516] Time since start: 10943.60s, Step: 14006, {'train/accuracy': 0.5588329081632653, 'train/loss': 3.752557248485332, 'validation/accuracy': 0.52142, 'validation/loss': 3.3949671875, 'validation/num_examples': 50000, 'test/accuracy': 0.4528, 'test/loss': 2.601360546875, 'test/num_examples': 10000, 'score': 10089.947731256485, 'total_duration': 10943.603086471558, 'accumulated_submission_time': 10089.947731256485, 'accumulated_eval_time': 829.4630072116852, 'accumulated_logging_time': 1.425502061843872} +I0913 05:43:08.901966 140490804545280 logging_writer.py:48] [14006] accumulated_eval_time=829.463, accumulated_logging_time=1.4255, accumulated_submission_time=10089.9, global_step=14006, preemption_count=0, score=10089.9, test/accuracy=0.4528, test/loss=2.60136, test/num_examples=10000, total_duration=10943.6, train/accuracy=0.558833, train/loss=3.75256, validation/accuracy=0.52142, validation/loss=3.39497, validation/num_examples=50000 +I0913 05:48:45.157326 140490745796352 logging_writer.py:48] [14500] global_step=14500, grad_norm=0.5, loss=1.85491 +I0913 05:48:45.161242 140514271229120 submission.py:307] 14500) loss = 1.855, grad_norm = 0.500 +I0913 05:57:33.428282 140490804545280 logging_writer.py:48] [15000] global_step=15000, grad_norm=0.5, loss=1.81329 +I0913 05:57:33.432085 140514271229120 submission.py:307] 15000) loss = 1.813, grad_norm = 0.500 +I0913 06:01:33.488718 140490745796352 logging_writer.py:48] [15500] global_step=15500, grad_norm=0.5, loss=1.68717 +I0913 06:01:33.493237 140514271229120 submission.py:307] 15500) loss = 1.687, grad_norm = 0.500 +I0913 06:08:59.938514 140490804545280 logging_writer.py:48] [16000] global_step=16000, grad_norm=0.5, loss=1.78031 +I0913 06:08:59.942601 140514271229120 submission.py:307] 16000) loss = 1.780, grad_norm = 0.500 +I0913 06:15:13.441918 140490745796352 logging_writer.py:48] [16500] global_step=16500, grad_norm=0.5, loss=1.67181 +I0913 06:15:13.446328 140514271229120 submission.py:307] 16500) loss = 1.672, grad_norm = 0.500 +I0913 06:16:25.836491 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 06:17:29.581389 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 06:18:34.050856 140514271229120 spec.py:363] Evaluating on the test split. +I0913 06:18:35.077382 140514271229120 submission_runner.py:516] Time since start: 13070.14s, Step: 16644, {'train/accuracy': 0.5981146364795918, 'train/loss': 3.6289943772919324, 'validation/accuracy': 0.54502, 'validation/loss': 3.5477440625, 'validation/num_examples': 50000, 'test/accuracy': 0.4807, 'test/loss': 2.3738681640625, 'test/num_examples': 10000, 'score': 12082.892327547073, 'total_duration': 13070.141052007675, 'accumulated_submission_time': 12082.892327547073, 'accumulated_eval_time': 958.7037978172302, 'accumulated_logging_time': 1.7975919246673584} +I0913 06:18:35.457105 140490754189056 logging_writer.py:48] [16644] accumulated_eval_time=958.704, accumulated_logging_time=1.79759, accumulated_submission_time=12082.9, global_step=16644, preemption_count=0, score=12082.9, test/accuracy=0.4807, test/loss=2.37387, test/num_examples=10000, total_duration=13070.1, train/accuracy=0.598115, train/loss=3.62899, validation/accuracy=0.54502, validation/loss=3.54774, validation/num_examples=50000 +I0913 06:22:56.661263 140490779367168 logging_writer.py:48] [17000] global_step=17000, grad_norm=0.5, loss=1.70214 +I0913 06:22:56.665381 140514271229120 submission.py:307] 17000) loss = 1.702, grad_norm = 0.500 +I0913 06:31:47.975159 140490754189056 logging_writer.py:48] [17500] global_step=17500, grad_norm=0.5, loss=1.74576 +I0913 06:31:47.994944 140514271229120 submission.py:307] 17500) loss = 1.746, grad_norm = 0.500 +I0913 06:35:55.675658 140490779367168 logging_writer.py:48] [18000] global_step=18000, grad_norm=0.5, loss=1.68968 +I0913 06:35:55.679596 140514271229120 submission.py:307] 18000) loss = 1.690, grad_norm = 0.500 +I0913 06:43:04.748586 140490754189056 logging_writer.py:48] [18500] global_step=18500, grad_norm=0.5, loss=1.70489 +I0913 06:43:04.765454 140514271229120 submission.py:307] 18500) loss = 1.705, grad_norm = 0.500 +I0913 06:49:20.182077 140490779367168 logging_writer.py:48] [19000] global_step=19000, grad_norm=0.5, loss=1.58799 +I0913 06:49:20.269416 140514271229120 submission.py:307] 19000) loss = 1.588, grad_norm = 0.500 +I0913 06:51:59.785952 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 06:52:58.662159 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 06:54:03.322181 140514271229120 spec.py:363] Evaluating on the test split. +I0913 06:54:04.365849 140514271229120 submission_runner.py:516] Time since start: 15199.43s, Step: 19287, {'train/accuracy': 0.5989516900510204, 'train/loss': 3.8001621791294644, 'validation/accuracy': 0.55098, 'validation/loss': 3.3991665625, 'validation/num_examples': 50000, 'test/accuracy': 0.4909, 'test/loss': 2.3680232421875, 'test/num_examples': 10000, 'score': 14081.901285886765, 'total_duration': 15199.429566860199, 'accumulated_submission_time': 14081.901285886765, 'accumulated_eval_time': 1083.2835838794708, 'accumulated_logging_time': 2.191770076751709} +I0913 06:54:04.658754 140490729010944 logging_writer.py:48] [19287] accumulated_eval_time=1083.28, accumulated_logging_time=2.19177, accumulated_submission_time=14081.9, global_step=19287, preemption_count=0, score=14081.9, test/accuracy=0.4909, test/loss=2.36802, test/num_examples=10000, total_duration=15199.4, train/accuracy=0.598952, train/loss=3.80016, validation/accuracy=0.55098, validation/loss=3.39917, validation/num_examples=50000 +I0913 06:56:32.469846 140490687047424 logging_writer.py:48] [19500] global_step=19500, grad_norm=0.5, loss=1.53224 +I0913 06:56:32.473807 140514271229120 submission.py:307] 19500) loss = 1.532, grad_norm = 0.500 +I0913 07:05:16.863202 140490729010944 logging_writer.py:48] [20000] global_step=20000, grad_norm=0.5, loss=1.59411 +I0913 07:05:16.867122 140514271229120 submission.py:307] 20000) loss = 1.594, grad_norm = 0.500 +I0913 07:09:27.149905 140490687047424 logging_writer.py:48] [20500] global_step=20500, grad_norm=0.5, loss=1.6251 +I0913 07:09:27.154415 140514271229120 submission.py:307] 20500) loss = 1.625, grad_norm = 0.500 +I0913 07:16:27.199449 140490729010944 logging_writer.py:48] [21000] global_step=21000, grad_norm=0.5, loss=1.48192 +I0913 07:16:27.203516 140514271229120 submission.py:307] 21000) loss = 1.482, grad_norm = 0.500 +I0913 07:22:41.952453 140490687047424 logging_writer.py:48] [21500] global_step=21500, grad_norm=0.5, loss=1.55874 +I0913 07:22:41.956993 140514271229120 submission.py:307] 21500) loss = 1.559, grad_norm = 0.500 +I0913 07:27:24.412387 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 07:28:23.470288 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 07:29:30.119948 140514271229120 spec.py:363] Evaluating on the test split. +I0913 07:29:31.029204 140514271229120 submission_runner.py:516] Time since start: 17326.09s, Step: 21970, {'train/accuracy': 0.5838647959183674, 'train/loss': 4.3787888507453765, 'validation/accuracy': 0.56536, 'validation/loss': 3.209695625, 'validation/num_examples': 50000, 'test/accuracy': 0.4916, 'test/loss': 2.3627771484375, 'test/num_examples': 10000, 'score': 16076.210381507874, 'total_duration': 17326.093100070953, 'accumulated_submission_time': 16076.210381507874, 'accumulated_eval_time': 1209.900592803955, 'accumulated_logging_time': 2.5014255046844482} +I0913 07:29:31.411974 140490779367168 logging_writer.py:48] [21970] accumulated_eval_time=1209.9, accumulated_logging_time=2.50143, accumulated_submission_time=16076.2, global_step=21970, preemption_count=0, score=16076.2, test/accuracy=0.4916, test/loss=2.36278, test/num_examples=10000, total_duration=17326.1, train/accuracy=0.583865, train/loss=4.37879, validation/accuracy=0.56536, validation/loss=3.2097, validation/num_examples=50000 +I0913 07:29:39.939911 140490687047424 logging_writer.py:48] [22000] global_step=22000, grad_norm=0.5, loss=1.54498 +I0913 07:29:39.943195 140514271229120 submission.py:307] 22000) loss = 1.545, grad_norm = 0.500 +I0913 07:37:39.591892 140490779367168 logging_writer.py:48] [22500] global_step=22500, grad_norm=0.5, loss=1.50854 +I0913 07:37:39.595929 140514271229120 submission.py:307] 22500) loss = 1.509, grad_norm = 0.500 +I0913 07:42:09.388218 140490687047424 logging_writer.py:48] [23000] global_step=23000, grad_norm=0.5, loss=1.47546 +I0913 07:42:09.392362 140514271229120 submission.py:307] 23000) loss = 1.475, grad_norm = 0.500 +I0913 07:49:07.360019 140490779367168 logging_writer.py:48] [23500] global_step=23500, grad_norm=0.5, loss=1.41013 +I0913 07:49:07.364073 140514271229120 submission.py:307] 23500) loss = 1.410, grad_norm = 0.500 +I0913 07:55:15.035100 140490687047424 logging_writer.py:48] [24000] global_step=24000, grad_norm=0.5, loss=1.45306 +I0913 07:55:15.047159 140514271229120 submission.py:307] 24000) loss = 1.453, grad_norm = 0.500 +I0913 08:00:20.253339 140490779367168 logging_writer.py:48] [24500] global_step=24500, grad_norm=0.5, loss=1.48735 +I0913 08:00:20.257691 140514271229120 submission.py:307] 24500) loss = 1.487, grad_norm = 0.500 +I0913 08:02:50.105879 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 08:03:42.457482 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 08:05:12.048861 140514271229120 spec.py:363] Evaluating on the test split. +I0913 08:05:12.955186 140514271229120 submission_runner.py:516] Time since start: 19468.02s, Step: 24669, {'train/accuracy': 0.640625, 'train/loss': 3.562229078643176, 'validation/accuracy': 0.56716, 'validation/loss': 3.4612678125, 'validation/num_examples': 50000, 'test/accuracy': 0.5159, 'test/loss': 2.26638359375, 'test/num_examples': 10000, 'score': 18070.13053560257, 'total_duration': 19468.019005060196, 'accumulated_submission_time': 18070.13053560257, 'accumulated_eval_time': 1352.7498168945312, 'accumulated_logging_time': 2.8937878608703613} +I0913 08:05:13.299580 140490712225536 logging_writer.py:48] [24669] accumulated_eval_time=1352.75, accumulated_logging_time=2.89379, accumulated_submission_time=18070.1, global_step=24669, preemption_count=0, score=18070.1, test/accuracy=0.5159, test/loss=2.26638, test/num_examples=10000, total_duration=19468, train/accuracy=0.640625, train/loss=3.56223, validation/accuracy=0.56716, validation/loss=3.46127, validation/num_examples=50000 +I0913 08:10:03.749129 140490829723392 logging_writer.py:48] [25000] global_step=25000, grad_norm=0.5, loss=1.50359 +I0913 08:10:03.753121 140514271229120 submission.py:307] 25000) loss = 1.504, grad_norm = 0.500 +I0913 08:14:40.202411 140490712225536 logging_writer.py:48] [25500] global_step=25500, grad_norm=0.5, loss=1.46283 +I0913 08:14:40.206752 140514271229120 submission.py:307] 25500) loss = 1.463, grad_norm = 0.500 +I0913 08:21:40.795794 140490829723392 logging_writer.py:48] [26000] global_step=26000, grad_norm=0.5, loss=1.40625 +I0913 08:21:40.799795 140514271229120 submission.py:307] 26000) loss = 1.406, grad_norm = 0.500 +I0913 08:27:57.163420 140490712225536 logging_writer.py:48] [26500] global_step=26500, grad_norm=0.5, loss=1.40622 +I0913 08:27:57.167995 140514271229120 submission.py:307] 26500) loss = 1.406, grad_norm = 0.500 +I0913 08:33:01.394740 140490829723392 logging_writer.py:48] [27000] global_step=27000, grad_norm=0.5, loss=1.43276 +I0913 08:33:01.399034 140514271229120 submission.py:307] 27000) loss = 1.433, grad_norm = 0.500 +I0913 08:38:32.365572 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 08:39:18.719125 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 08:40:46.438195 140514271229120 spec.py:363] Evaluating on the test split. +I0913 08:40:47.482569 140514271229120 submission_runner.py:516] Time since start: 21602.55s, Step: 27331, {'train/accuracy': 0.6507095025510204, 'train/loss': 3.3582365074936225, 'validation/accuracy': 0.5733, 'validation/loss': 3.3512609375, 'validation/num_examples': 50000, 'test/accuracy': 0.5157, 'test/loss': 2.247359765625, 'test/num_examples': 10000, 'score': 20065.606693983078, 'total_duration': 21602.546391010284, 'accumulated_submission_time': 20065.606693983078, 'accumulated_eval_time': 1487.866794347763, 'accumulated_logging_time': 3.247710704803467} +I0913 08:40:47.882325 140490695440128 logging_writer.py:48] [27331] accumulated_eval_time=1487.87, accumulated_logging_time=3.24771, accumulated_submission_time=20065.6, global_step=27331, preemption_count=0, score=20065.6, test/accuracy=0.5157, test/loss=2.24736, test/num_examples=10000, total_duration=21602.5, train/accuracy=0.65071, train/loss=3.35824, validation/accuracy=0.5733, validation/loss=3.35126, validation/num_examples=50000 +I0913 08:42:47.085214 140490754189056 logging_writer.py:48] [27500] global_step=27500, grad_norm=0.5, loss=1.40828 +I0913 08:42:47.089226 140514271229120 submission.py:307] 27500) loss = 1.408, grad_norm = 0.500 +I0913 08:47:25.244088 140490695440128 logging_writer.py:48] [28000] global_step=28000, grad_norm=0.5, loss=1.29432 +I0913 08:47:25.248098 140514271229120 submission.py:307] 28000) loss = 1.294, grad_norm = 0.500 +I0913 08:54:39.643395 140490754189056 logging_writer.py:48] [28500] global_step=28500, grad_norm=0.5, loss=1.42535 +I0913 08:54:39.647341 140514271229120 submission.py:307] 28500) loss = 1.425, grad_norm = 0.500 +I0913 09:00:56.621377 140490695440128 logging_writer.py:48] [29000] global_step=29000, grad_norm=0.5, loss=1.24399 +I0913 09:00:56.625424 140514271229120 submission.py:307] 29000) loss = 1.244, grad_norm = 0.500 +I0913 09:06:04.466514 140490754189056 logging_writer.py:48] [29500] global_step=29500, grad_norm=0.5, loss=1.3387 +I0913 09:06:04.470563 140514271229120 submission.py:307] 29500) loss = 1.339, grad_norm = 0.500 +I0913 09:14:07.556254 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 09:14:52.158849 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 09:15:57.744463 140514271229120 spec.py:363] Evaluating on the test split. +I0913 09:15:58.734719 140514271229120 submission_runner.py:516] Time since start: 23713.80s, Step: 29969, {'train/accuracy': 0.6632852359693877, 'train/loss': 3.6978236607142856, 'validation/accuracy': 0.59528, 'validation/loss': 3.07289625, 'validation/num_examples': 50000, 'test/accuracy': 0.5312, 'test/loss': 2.207117578125, 'test/num_examples': 10000, 'score': 22061.40389227867, 'total_duration': 23713.798048257828, 'accumulated_submission_time': 22061.40389227867, 'accumulated_eval_time': 1599.0447170734406, 'accumulated_logging_time': 3.656982898712158} +I0913 09:15:59.180780 140490712225536 logging_writer.py:48] [29969] accumulated_eval_time=1599.04, accumulated_logging_time=3.65698, accumulated_submission_time=22061.4, global_step=29969, preemption_count=0, score=22061.4, test/accuracy=0.5312, test/loss=2.20712, test/num_examples=10000, total_duration=23713.8, train/accuracy=0.663285, train/loss=3.69782, validation/accuracy=0.59528, validation/loss=3.0729, validation/num_examples=50000 +I0913 09:16:09.430970 140490745796352 logging_writer.py:48] [30000] global_step=30000, grad_norm=0.5, loss=1.26641 +I0913 09:16:09.434275 140514271229120 submission.py:307] 30000) loss = 1.266, grad_norm = 0.500 +I0913 09:21:16.411065 140490712225536 logging_writer.py:48] [30500] global_step=30500, grad_norm=0.5, loss=1.31472 +I0913 09:21:16.415311 140514271229120 submission.py:307] 30500) loss = 1.315, grad_norm = 0.500 +I0913 09:28:48.879542 140490745796352 logging_writer.py:48] [31000] global_step=31000, grad_norm=0.5, loss=1.22232 +I0913 09:28:48.883728 140514271229120 submission.py:307] 31000) loss = 1.222, grad_norm = 0.500 +I0913 09:35:09.172668 140490712225536 logging_writer.py:48] [31500] global_step=31500, grad_norm=0.5, loss=1.216 +I0913 09:35:09.181572 140514271229120 submission.py:307] 31500) loss = 1.216, grad_norm = 0.500 +I0913 09:40:20.037790 140490745796352 logging_writer.py:48] [32000] global_step=32000, grad_norm=0.5, loss=1.29962 +I0913 09:40:20.041843 140514271229120 submission.py:307] 32000) loss = 1.300, grad_norm = 0.500 +I0913 09:48:59.404197 140490712225536 logging_writer.py:48] [32500] global_step=32500, grad_norm=0.5, loss=1.47295 +I0913 09:48:59.408235 140514271229120 submission.py:307] 32500) loss = 1.473, grad_norm = 0.500 +I0913 09:49:18.014011 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 09:50:13.217057 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 09:51:17.749633 140514271229120 spec.py:363] Evaluating on the test split. +I0913 09:51:18.752874 140514271229120 submission_runner.py:516] Time since start: 25833.82s, Step: 32527, {'train/accuracy': 0.6575454400510204, 'train/loss': 4.162380296356824, 'validation/accuracy': 0.58764, 'validation/loss': 3.505524375, 'validation/num_examples': 50000, 'test/accuracy': 0.5319, 'test/loss': 2.2375390625, 'test/num_examples': 10000, 'score': 24056.594072580338, 'total_duration': 25833.816573143005, 'accumulated_submission_time': 24056.594072580338, 'accumulated_eval_time': 1719.7836360931396, 'accumulated_logging_time': 4.1150946617126465} +I0913 09:51:19.182166 140490737403648 logging_writer.py:48] [32527] accumulated_eval_time=1719.78, accumulated_logging_time=4.11509, accumulated_submission_time=24056.6, global_step=32527, preemption_count=0, score=24056.6, test/accuracy=0.5319, test/loss=2.23754, test/num_examples=10000, total_duration=25833.8, train/accuracy=0.657545, train/loss=4.16238, validation/accuracy=0.58764, validation/loss=3.50552, validation/num_examples=50000 +I0913 09:56:01.548343 140490695440128 logging_writer.py:48] [33000] global_step=33000, grad_norm=0.5, loss=1.27631 +I0913 09:56:01.552393 140514271229120 submission.py:307] 33000) loss = 1.276, grad_norm = 0.500 +I0913 10:03:39.466926 140490737403648 logging_writer.py:48] [33500] global_step=33500, grad_norm=0.5, loss=1.28788 +I0913 10:03:39.470952 140514271229120 submission.py:307] 33500) loss = 1.288, grad_norm = 0.500 +I0913 10:10:01.229971 140490695440128 logging_writer.py:48] [34000] global_step=34000, grad_norm=0.5, loss=1.17779 +I0913 10:10:01.233931 140514271229120 submission.py:307] 34000) loss = 1.178, grad_norm = 0.500 +I0913 10:15:06.325137 140490737403648 logging_writer.py:48] [34500] global_step=34500, grad_norm=0.5, loss=1.28291 +I0913 10:15:06.329840 140514271229120 submission.py:307] 34500) loss = 1.283, grad_norm = 0.500 +I0913 10:23:50.008804 140490695440128 logging_writer.py:48] [35000] global_step=35000, grad_norm=0.5, loss=1.15369 +I0913 10:23:50.012844 140514271229120 submission.py:307] 35000) loss = 1.154, grad_norm = 0.500 +I0913 10:24:35.937902 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 10:25:32.391260 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 10:26:36.638524 140514271229120 spec.py:363] Evaluating on the test split. +I0913 10:26:37.698499 140514271229120 submission_runner.py:516] Time since start: 27952.76s, Step: 35088, {'train/accuracy': 0.6999362244897959, 'train/loss': 3.3552706971460458, 'validation/accuracy': 0.59926, 'validation/loss': 3.23747375, 'validation/num_examples': 50000, 'test/accuracy': 0.5413, 'test/loss': 2.1987033203125, 'test/num_examples': 10000, 'score': 26049.596282482147, 'total_duration': 27952.762268304825, 'accumulated_submission_time': 26049.596282482147, 'accumulated_eval_time': 1841.544192790985, 'accumulated_logging_time': 4.55816388130188} +I0913 10:26:38.298849 140490737403648 logging_writer.py:48] [35088] accumulated_eval_time=1841.54, accumulated_logging_time=4.55816, accumulated_submission_time=26049.6, global_step=35088, preemption_count=0, score=26049.6, test/accuracy=0.5413, test/loss=2.1987, test/num_examples=10000, total_duration=27952.8, train/accuracy=0.699936, train/loss=3.35527, validation/accuracy=0.59926, validation/loss=3.23747, validation/num_examples=50000 +I0913 10:30:44.930711 140490821330688 logging_writer.py:48] [35500] global_step=35500, grad_norm=0.5, loss=1.23226 +I0913 10:30:44.934867 140514271229120 submission.py:307] 35500) loss = 1.232, grad_norm = 0.500 +I0913 10:38:24.545462 140490737403648 logging_writer.py:48] [36000] global_step=36000, grad_norm=0.5, loss=1.18951 +I0913 10:38:24.549424 140514271229120 submission.py:307] 36000) loss = 1.190, grad_norm = 0.500 +I0913 10:44:45.356037 140490821330688 logging_writer.py:48] [36500] global_step=36500, grad_norm=0.5, loss=1.196 +I0913 10:44:45.361459 140514271229120 submission.py:307] 36500) loss = 1.196, grad_norm = 0.500 +I0913 10:49:48.886587 140490737403648 logging_writer.py:48] [37000] global_step=37000, grad_norm=0.5, loss=1.2235 +I0913 10:49:48.890538 140514271229120 submission.py:307] 37000) loss = 1.224, grad_norm = 0.500 +I0913 10:58:32.504024 140490821330688 logging_writer.py:48] [37500] global_step=37500, grad_norm=0.5, loss=1.18637 +I0913 10:58:32.508073 140514271229120 submission.py:307] 37500) loss = 1.186, grad_norm = 0.500 +I0913 10:59:54.948084 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 11:00:53.080099 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 11:01:57.881484 140514271229120 spec.py:363] Evaluating on the test split. +I0913 11:01:58.923488 140514271229120 submission_runner.py:516] Time since start: 30073.99s, Step: 37681, {'train/accuracy': 0.7167171556122449, 'train/loss': 3.406367710658482, 'validation/accuracy': 0.60236, 'validation/loss': 3.4714803125, 'validation/num_examples': 50000, 'test/accuracy': 0.527, 'test/loss': 2.26468828125, 'test/num_examples': 10000, 'score': 28042.45645093918, 'total_duration': 30073.98717403412, 'accumulated_submission_time': 28042.45645093918, 'accumulated_eval_time': 1965.5194764137268, 'accumulated_logging_time': 5.167999982833862} +I0913 11:01:59.287357 140490745796352 logging_writer.py:48] [37681] accumulated_eval_time=1965.52, accumulated_logging_time=5.168, accumulated_submission_time=28042.5, global_step=37681, preemption_count=0, score=28042.5, test/accuracy=0.527, test/loss=2.26469, test/num_examples=10000, total_duration=30074, train/accuracy=0.716717, train/loss=3.40637, validation/accuracy=0.60236, validation/loss=3.47148, validation/num_examples=50000 +I0913 11:04:57.491297 140490787759872 logging_writer.py:48] [38000] global_step=38000, grad_norm=0.5, loss=1.10126 +I0913 11:04:57.495403 140514271229120 submission.py:307] 38000) loss = 1.101, grad_norm = 0.500 +I0913 11:12:25.549341 140490745796352 logging_writer.py:48] [38500] global_step=38500, grad_norm=0.5, loss=1.18672 +I0913 11:12:25.553559 140514271229120 submission.py:307] 38500) loss = 1.187, grad_norm = 0.500 +I0913 11:18:58.657116 140490787759872 logging_writer.py:48] [39000] global_step=39000, grad_norm=0.5, loss=1.11047 +I0913 11:18:58.668172 140514271229120 submission.py:307] 39000) loss = 1.110, grad_norm = 0.500 +I0913 11:23:52.140328 140490745796352 logging_writer.py:48] [39500] global_step=39500, grad_norm=0.5, loss=1.17192 +I0913 11:23:52.144305 140514271229120 submission.py:307] 39500) loss = 1.172, grad_norm = 0.500 +I0913 11:32:31.619805 140490787759872 logging_writer.py:48] [40000] global_step=40000, grad_norm=0.5, loss=1.28691 +I0913 11:32:31.623844 140514271229120 submission.py:307] 40000) loss = 1.287, grad_norm = 0.500 +I0913 11:35:16.155821 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 11:36:20.292074 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 11:37:24.737563 140514271229120 spec.py:363] Evaluating on the test split. +I0913 11:37:25.738372 140514271229120 submission_runner.py:516] Time since start: 32200.80s, Step: 40351, {'train/accuracy': 0.7042012117346939, 'train/loss': 3.5335310332629146, 'validation/accuracy': 0.58318, 'validation/loss': 3.826075, 'validation/num_examples': 50000, 'test/accuracy': 0.5265, 'test/loss': 2.308673828125, 'test/num_examples': 10000, 'score': 30035.080238819122, 'total_duration': 32200.802064180374, 'accumulated_submission_time': 30035.080238819122, 'accumulated_eval_time': 2095.101897716522, 'accumulated_logging_time': 5.541431188583374} +I0913 11:37:26.085302 140490812937984 logging_writer.py:48] [40351] accumulated_eval_time=2095.1, accumulated_logging_time=5.54143, accumulated_submission_time=30035.1, global_step=40351, preemption_count=0, score=30035.1, test/accuracy=0.5265, test/loss=2.30867, test/num_examples=10000, total_duration=32200.8, train/accuracy=0.704201, train/loss=3.53353, validation/accuracy=0.58318, validation/loss=3.82607, validation/num_examples=50000 +I0913 11:38:40.288475 140490720618240 logging_writer.py:48] [40500] global_step=40500, grad_norm=0.5, loss=1.22247 +I0913 11:38:40.292255 140514271229120 submission.py:307] 40500) loss = 1.222, grad_norm = 0.500 +I0913 11:46:25.153447 140490812937984 logging_writer.py:48] [41000] global_step=41000, grad_norm=0.5, loss=1.14579 +I0913 11:46:25.157534 140514271229120 submission.py:307] 41000) loss = 1.146, grad_norm = 0.500 +I0913 11:53:08.949381 140490720618240 logging_writer.py:48] [41500] global_step=41500, grad_norm=0.5, loss=1.2026 +I0913 11:53:08.953533 140514271229120 submission.py:307] 41500) loss = 1.203, grad_norm = 0.500 +I0913 11:58:03.520092 140490812937984 logging_writer.py:48] [42000] global_step=42000, grad_norm=0.5, loss=1.14367 +I0913 11:58:03.525920 140514271229120 submission.py:307] 42000) loss = 1.144, grad_norm = 0.500 +I0913 12:06:42.458129 140490720618240 logging_writer.py:48] [42500] global_step=42500, grad_norm=0.5, loss=1.11701 +I0913 12:06:42.462118 140514271229120 submission.py:307] 42500) loss = 1.117, grad_norm = 0.500 +I0913 12:10:43.003499 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 12:11:47.433442 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 12:12:52.331775 140514271229120 spec.py:363] Evaluating on the test split. +I0913 12:12:53.394586 140514271229120 submission_runner.py:516] Time since start: 34328.46s, Step: 42975, {'train/accuracy': 0.7175143494897959, 'train/loss': 3.674822748923788, 'validation/accuracy': 0.60794, 'validation/loss': 3.3489609375, 'validation/num_examples': 50000, 'test/accuracy': 0.5359, 'test/loss': 2.2669248046875, 'test/num_examples': 10000, 'score': 32028.396780967712, 'total_duration': 34328.45838928223, 'accumulated_submission_time': 32028.396780967712, 'accumulated_eval_time': 2225.492896080017, 'accumulated_logging_time': 5.898090839385986} +I0913 12:12:53.802839 140490720618240 logging_writer.py:48] [42975] accumulated_eval_time=2225.49, accumulated_logging_time=5.89809, accumulated_submission_time=32028.4, global_step=42975, preemption_count=0, score=32028.4, test/accuracy=0.5359, test/loss=2.26692, test/num_examples=10000, total_duration=34328.5, train/accuracy=0.717514, train/loss=3.67482, validation/accuracy=0.60794, validation/loss=3.34896, validation/num_examples=50000 +I0913 12:13:02.956012 140490687047424 logging_writer.py:48] [43000] global_step=43000, grad_norm=0.5, loss=1.25442 +I0913 12:13:02.960699 140514271229120 submission.py:307] 43000) loss = 1.254, grad_norm = 0.500 +I0913 12:20:15.418101 140490720618240 logging_writer.py:48] [43500] global_step=43500, grad_norm=0.5, loss=1.15407 +I0913 12:20:15.422089 140514271229120 submission.py:307] 43500) loss = 1.154, grad_norm = 0.500 +I0913 12:27:01.676287 140490687047424 logging_writer.py:48] [44000] global_step=44000, grad_norm=0.5, loss=1.06136 +I0913 12:27:01.680710 140514271229120 submission.py:307] 44000) loss = 1.061, grad_norm = 0.500 +I0913 12:31:57.699346 140490720618240 logging_writer.py:48] [44500] global_step=44500, grad_norm=0.5, loss=1.03335 +I0913 12:31:57.703488 140514271229120 submission.py:307] 44500) loss = 1.033, grad_norm = 0.500 +I0913 12:40:32.132658 140490687047424 logging_writer.py:48] [45000] global_step=45000, grad_norm=0.5, loss=1.03396 +I0913 12:40:32.136807 140514271229120 submission.py:307] 45000) loss = 1.034, grad_norm = 0.500 +I0913 12:44:44.683324 140490720618240 logging_writer.py:48] [45500] global_step=45500, grad_norm=0.5, loss=1.04281 +I0913 12:44:44.687798 140514271229120 submission.py:307] 45500) loss = 1.043, grad_norm = 0.500 +I0913 12:46:14.242289 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 12:47:13.337741 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 12:48:18.529612 140514271229120 spec.py:363] Evaluating on the test split. +I0913 12:48:19.548755 140514271229120 submission_runner.py:516] Time since start: 36454.61s, Step: 45639, {'train/accuracy': 0.7144650829081632, 'train/loss': 3.7727309246452485, 'validation/accuracy': 0.60726, 'validation/loss': 3.3839653125, 'validation/num_examples': 50000, 'test/accuracy': 0.5427, 'test/loss': 2.2691771484375, 'test/num_examples': 10000, 'score': 34023.23478484154, 'total_duration': 36454.61201810837, 'accumulated_submission_time': 34023.23478484154, 'accumulated_eval_time': 2350.798717737198, 'accumulated_logging_time': 6.316791296005249} +I0913 12:48:19.930963 140490678654720 logging_writer.py:48] [45639] accumulated_eval_time=2350.8, accumulated_logging_time=6.31679, accumulated_submission_time=34023.2, global_step=45639, preemption_count=0, score=34023.2, test/accuracy=0.5427, test/loss=2.26918, test/num_examples=10000, total_duration=36454.6, train/accuracy=0.714465, train/loss=3.77273, validation/accuracy=0.60726, validation/loss=3.38397, validation/num_examples=50000 +I0913 12:53:26.857115 140490779367168 logging_writer.py:48] [46000] global_step=46000, grad_norm=0.5, loss=1.17822 +I0913 12:53:26.860922 140514271229120 submission.py:307] 46000) loss = 1.178, grad_norm = 0.500 +I0913 13:00:09.418411 140490678654720 logging_writer.py:48] [46500] global_step=46500, grad_norm=0.5, loss=1.11525 +I0913 13:00:09.466844 140514271229120 submission.py:307] 46500) loss = 1.115, grad_norm = 0.500 +I0913 13:05:19.403697 140490779367168 logging_writer.py:48] [47000] global_step=47000, grad_norm=0.5, loss=1.11806 +I0913 13:05:19.407964 140514271229120 submission.py:307] 47000) loss = 1.118, grad_norm = 0.500 +I0913 13:13:53.742215 140490678654720 logging_writer.py:48] [47500] global_step=47500, grad_norm=0.5, loss=1.06852 +I0913 13:13:53.746694 140514271229120 submission.py:307] 47500) loss = 1.069, grad_norm = 0.500 +I0913 13:17:56.239594 140490779367168 logging_writer.py:48] [48000] global_step=48000, grad_norm=0.5, loss=1.01294 +I0913 13:17:56.243827 140514271229120 submission.py:307] 48000) loss = 1.013, grad_norm = 0.500 +I0913 13:21:38.340524 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 13:22:37.630360 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 13:23:50.397386 140514271229120 spec.py:363] Evaluating on the test split. +I0913 13:23:51.390160 140514271229120 submission_runner.py:516] Time since start: 38586.45s, Step: 48310, {'train/accuracy': 0.7372249681122449, 'train/loss': 3.4263698032924106, 'validation/accuracy': 0.59002, 'validation/loss': 3.86083, 'validation/num_examples': 50000, 'test/accuracy': 0.5378, 'test/loss': 2.2942724609375, 'test/num_examples': 10000, 'score': 36016.49505734444, 'total_duration': 38586.453929424286, 'accumulated_submission_time': 36016.49505734444, 'accumulated_eval_time': 2483.848254919052, 'accumulated_logging_time': 6.711942195892334} +I0913 13:23:51.783279 140490804545280 logging_writer.py:48] [48310] accumulated_eval_time=2483.85, accumulated_logging_time=6.71194, accumulated_submission_time=36016.5, global_step=48310, preemption_count=0, score=36016.5, test/accuracy=0.5378, test/loss=2.29427, test/num_examples=10000, total_duration=38586.5, train/accuracy=0.737225, train/loss=3.42637, validation/accuracy=0.59002, validation/loss=3.86083, validation/num_examples=50000 +I0913 13:26:02.874547 140490745796352 logging_writer.py:48] [48500] global_step=48500, grad_norm=0.5, loss=0.991121 +I0913 13:26:02.878422 140514271229120 submission.py:307] 48500) loss = 0.991, grad_norm = 0.500 +I0913 13:32:50.653748 140490804545280 logging_writer.py:48] [49000] global_step=49000, grad_norm=0.5, loss=1.02664 +I0913 13:32:50.670954 140514271229120 submission.py:307] 49000) loss = 1.027, grad_norm = 0.500 +I0913 13:38:00.800691 140490745796352 logging_writer.py:48] [49500] global_step=49500, grad_norm=0.5, loss=1.06455 +I0913 13:38:00.804644 140514271229120 submission.py:307] 49500) loss = 1.065, grad_norm = 0.500 +I0913 13:46:31.677124 140490804545280 logging_writer.py:48] [50000] global_step=50000, grad_norm=0.5, loss=1.04505 +I0913 13:46:31.681158 140514271229120 submission.py:307] 50000) loss = 1.045, grad_norm = 0.500 +I0913 13:50:37.467808 140490745796352 logging_writer.py:48] [50500] global_step=50500, grad_norm=0.5, loss=0.952922 +I0913 13:50:37.472993 140514271229120 submission.py:307] 50500) loss = 0.953, grad_norm = 0.500 +I0913 13:57:13.176825 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 13:58:02.601379 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 13:59:23.976657 140514271229120 spec.py:363] Evaluating on the test split. +I0913 13:59:24.915154 140514271229120 submission_runner.py:516] Time since start: 40719.98s, Step: 50983, {'train/accuracy': 0.7030253507653061, 'train/loss': 4.454295878507653, 'validation/accuracy': 0.60278, 'validation/loss': 3.635655625, 'validation/num_examples': 50000, 'test/accuracy': 0.5423, 'test/loss': 2.321203125, 'test/num_examples': 10000, 'score': 38012.98178291321, 'total_duration': 40719.979044914246, 'accumulated_submission_time': 38012.98178291321, 'accumulated_eval_time': 2615.5865807533264, 'accumulated_logging_time': 7.114745140075684} +I0913 13:59:25.292883 140490712225536 logging_writer.py:48] [50983] accumulated_eval_time=2615.59, accumulated_logging_time=7.11475, accumulated_submission_time=38013, global_step=50983, preemption_count=0, score=38013, test/accuracy=0.5423, test/loss=2.3212, test/num_examples=10000, total_duration=40720, train/accuracy=0.703025, train/loss=4.4543, validation/accuracy=0.60278, validation/loss=3.63566, validation/num_examples=50000 +I0913 13:59:30.601804 140490678654720 logging_writer.py:48] [51000] global_step=51000, grad_norm=0.5, loss=1.03725 +I0913 13:59:30.605170 140514271229120 submission.py:307] 51000) loss = 1.037, grad_norm = 0.500 +I0913 14:05:29.177432 140490712225536 logging_writer.py:48] [51500] global_step=51500, grad_norm=0.5, loss=0.977114 +I0913 14:05:29.181581 140514271229120 submission.py:307] 51500) loss = 0.977, grad_norm = 0.500 +I0913 14:10:57.842576 140490678654720 logging_writer.py:48] [52000] global_step=52000, grad_norm=0.5, loss=1.07618 +I0913 14:10:57.846714 140514271229120 submission.py:307] 52000) loss = 1.076, grad_norm = 0.500 +I0913 14:19:20.014428 140490712225536 logging_writer.py:48] [52500] global_step=52500, grad_norm=0.5, loss=1.08427 +I0913 14:19:20.018396 140514271229120 submission.py:307] 52500) loss = 1.084, grad_norm = 0.500 +I0913 14:23:20.081612 140490678654720 logging_writer.py:48] [53000] global_step=53000, grad_norm=0.5, loss=0.940408 +I0913 14:23:20.085797 140514271229120 submission.py:307] 53000) loss = 0.940, grad_norm = 0.500 +I0913 14:30:07.804182 140490712225536 logging_writer.py:48] [53500] global_step=53500, grad_norm=0.5, loss=1.03496 +I0913 14:30:07.808342 140514271229120 submission.py:307] 53500) loss = 1.035, grad_norm = 0.500 +I0913 14:32:56.385947 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 14:33:45.440374 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 14:35:11.410725 140514271229120 spec.py:363] Evaluating on the test split. +I0913 14:35:12.316162 140514271229120 submission_runner.py:516] Time since start: 42867.38s, Step: 53643, {'train/accuracy': 0.7354512117346939, 'train/loss': 3.9732946279097576, 'validation/accuracy': 0.60552, 'validation/loss': 3.769000625, 'validation/num_examples': 50000, 'test/accuracy': 0.5485, 'test/loss': 2.3113056640625, 'test/num_examples': 10000, 'score': 40020.475558280945, 'total_duration': 42867.38005900383, 'accumulated_submission_time': 40020.475558280945, 'accumulated_eval_time': 2751.5167932510376, 'accumulated_logging_time': 7.502183437347412} +I0913 14:35:12.772668 140490796152576 logging_writer.py:48] [53643] accumulated_eval_time=2751.52, accumulated_logging_time=7.50218, accumulated_submission_time=40020.5, global_step=53643, preemption_count=0, score=40020.5, test/accuracy=0.5485, test/loss=2.31131, test/num_examples=10000, total_duration=42867.4, train/accuracy=0.735451, train/loss=3.97329, validation/accuracy=0.60552, validation/loss=3.769, validation/num_examples=50000 +I0913 14:38:39.500073 140490712225536 logging_writer.py:48] [54000] global_step=54000, grad_norm=0.5, loss=1.10732 +I0913 14:38:39.504181 140514271229120 submission.py:307] 54000) loss = 1.107, grad_norm = 0.500 +I0913 14:44:17.392412 140490796152576 logging_writer.py:48] [54500] global_step=54500, grad_norm=0.5, loss=1.02061 +I0913 14:44:17.396442 140514271229120 submission.py:307] 54500) loss = 1.021, grad_norm = 0.500 +I0913 14:52:46.040962 140490712225536 logging_writer.py:48] [55000] global_step=55000, grad_norm=0.5, loss=1.05969 +I0913 14:52:46.045079 140514271229120 submission.py:307] 55000) loss = 1.060, grad_norm = 0.500 +I0913 14:56:44.784387 140490796152576 logging_writer.py:48] [55500] global_step=55500, grad_norm=0.5, loss=0.909215 +I0913 14:56:44.788537 140514271229120 submission.py:307] 55500) loss = 0.909, grad_norm = 0.500 +I0913 15:03:45.562390 140490712225536 logging_writer.py:48] [56000] global_step=56000, grad_norm=0.5, loss=0.908519 +I0913 15:03:45.566413 140514271229120 submission.py:307] 56000) loss = 0.909, grad_norm = 0.500 +I0913 15:08:29.426622 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 15:09:00.042680 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 15:10:04.511301 140514271229120 spec.py:363] Evaluating on the test split. +I0913 15:10:05.547141 140514271229120 submission_runner.py:516] Time since start: 44960.61s, Step: 56265, {'train/accuracy': 0.7109375, 'train/loss': 4.522355994399713, 'validation/accuracy': 0.60638, 'validation/loss': 3.6884934375, 'validation/num_examples': 50000, 'test/accuracy': 0.5361, 'test/loss': 2.37835546875, 'test/num_examples': 10000, 'score': 42013.84645605087, 'total_duration': 44960.609933137894, 'accumulated_submission_time': 42013.84645605087, 'accumulated_eval_time': 2847.636313676834, 'accumulated_logging_time': 7.968269109725952} +I0913 15:10:05.957314 140490745796352 logging_writer.py:48] [56265] accumulated_eval_time=2847.64, accumulated_logging_time=7.96827, accumulated_submission_time=42013.8, global_step=56265, preemption_count=0, score=42013.8, test/accuracy=0.5361, test/loss=2.37836, test/num_examples=10000, total_duration=44960.6, train/accuracy=0.710938, train/loss=4.52236, validation/accuracy=0.60638, validation/loss=3.68849, validation/num_examples=50000 +I0913 15:12:45.903038 140490821330688 logging_writer.py:48] [56500] global_step=56500, grad_norm=0.5, loss=0.870556 +I0913 15:12:45.907051 140514271229120 submission.py:307] 56500) loss = 0.871, grad_norm = 0.500 +I0913 15:18:39.402636 140490745796352 logging_writer.py:48] [57000] global_step=57000, grad_norm=0.5, loss=0.899343 +I0913 15:18:39.406691 140514271229120 submission.py:307] 57000) loss = 0.899, grad_norm = 0.500 +I0913 15:27:28.507301 140490821330688 logging_writer.py:48] [57500] global_step=57500, grad_norm=0.5, loss=0.984226 +I0913 15:27:28.511239 140514271229120 submission.py:307] 57500) loss = 0.984, grad_norm = 0.500 +I0913 15:31:28.128063 140490745796352 logging_writer.py:48] [58000] global_step=58000, grad_norm=0.5, loss=0.901752 +I0913 15:31:28.132397 140514271229120 submission.py:307] 58000) loss = 0.902, grad_norm = 0.500 +I0913 15:38:35.886041 140490821330688 logging_writer.py:48] [58500] global_step=58500, grad_norm=0.5, loss=1.05591 +I0913 15:38:35.890228 140514271229120 submission.py:307] 58500) loss = 1.056, grad_norm = 0.500 +I0913 15:43:24.106276 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 15:44:01.755679 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 15:45:06.354487 140514271229120 spec.py:363] Evaluating on the test split. +I0913 15:45:07.383605 140514271229120 submission_runner.py:516] Time since start: 47062.45s, Step: 58754, {'train/accuracy': 0.7159996811224489, 'train/loss': 4.573967135682398, 'validation/accuracy': 0.5886, 'validation/loss': 4.122043125, 'validation/num_examples': 50000, 'test/accuracy': 0.5422, 'test/loss': 2.350933203125, 'test/num_examples': 10000, 'score': 44008.71176171303, 'total_duration': 47062.44623923302, 'accumulated_submission_time': 44008.71176171303, 'accumulated_eval_time': 2950.9123871326447, 'accumulated_logging_time': 8.387396335601807} +I0913 15:45:07.797709 140490762581760 logging_writer.py:48] [58754] accumulated_eval_time=2950.91, accumulated_logging_time=8.3874, accumulated_submission_time=44008.7, global_step=58754, preemption_count=0, score=44008.7, test/accuracy=0.5422, test/loss=2.35093, test/num_examples=10000, total_duration=47062.4, train/accuracy=0.716, train/loss=4.57397, validation/accuracy=0.5886, validation/loss=4.12204, validation/num_examples=50000 +I0913 15:47:47.248306 140494889809664 logging_writer.py:48] [59000] global_step=59000, grad_norm=0.5, loss=0.992041 +I0913 15:47:47.252091 140514271229120 submission.py:307] 59000) loss = 0.992, grad_norm = 0.500 +I0913 15:53:44.905048 140490762581760 logging_writer.py:48] [59500] global_step=59500, grad_norm=0.5, loss=0.936414 +I0913 15:53:44.909058 140514271229120 submission.py:307] 59500) loss = 0.936, grad_norm = 0.500 +I0913 16:02:25.108294 140494889809664 logging_writer.py:48] [60000] global_step=60000, grad_norm=0.5, loss=1.02102 +I0913 16:02:25.112407 140514271229120 submission.py:307] 60000) loss = 1.021, grad_norm = 0.500 +I0913 16:06:28.893058 140490762581760 logging_writer.py:48] [60500] global_step=60500, grad_norm=0.5, loss=0.999245 +I0913 16:06:28.897445 140514271229120 submission.py:307] 60500) loss = 0.999, grad_norm = 0.500 +I0913 16:13:29.619872 140494889809664 logging_writer.py:48] [61000] global_step=61000, grad_norm=0.5, loss=0.967177 +I0913 16:13:29.623911 140514271229120 submission.py:307] 61000) loss = 0.967, grad_norm = 0.500 +I0913 16:18:29.296096 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 16:19:00.248637 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 16:20:04.493580 140514271229120 spec.py:363] Evaluating on the test split. +I0913 16:20:05.539877 140514271229120 submission_runner.py:516] Time since start: 49160.60s, Step: 61268, {'train/accuracy': 0.7661830357142857, 'train/loss': 3.472351697026467, 'validation/accuracy': 0.59942, 'validation/loss': 4.078488125, 'validation/num_examples': 50000, 'test/accuracy': 0.5413, 'test/loss': 2.3884078125, 'test/num_examples': 10000, 'score': 46006.778717279434, 'total_duration': 49160.60372567177, 'accumulated_submission_time': 46006.778717279434, 'accumulated_eval_time': 3047.156206846237, 'accumulated_logging_time': 8.810739994049072} +I0913 16:20:05.978824 140490712225536 logging_writer.py:48] [61268] accumulated_eval_time=3047.16, accumulated_logging_time=8.81074, accumulated_submission_time=46006.8, global_step=61268, preemption_count=0, score=46006.8, test/accuracy=0.5413, test/loss=2.38841, test/num_examples=10000, total_duration=49160.6, train/accuracy=0.766183, train/loss=3.47235, validation/accuracy=0.59942, validation/loss=4.07849, validation/num_examples=50000 +I0913 16:22:28.133943 140490737403648 logging_writer.py:48] [61500] global_step=61500, grad_norm=0.5, loss=0.832256 +I0913 16:22:28.137806 140514271229120 submission.py:307] 61500) loss = 0.832, grad_norm = 0.500 +I0913 16:28:21.761755 140490712225536 logging_writer.py:48] [62000] global_step=62000, grad_norm=0.5, loss=0.923117 +I0913 16:28:21.766032 140514271229120 submission.py:307] 62000) loss = 0.923, grad_norm = 0.500 +I0913 16:37:01.079574 140490737403648 logging_writer.py:48] [62500] global_step=62500, grad_norm=0.5, loss=1.0612 +I0913 16:37:01.083717 140514271229120 submission.py:307] 62500) loss = 1.061, grad_norm = 0.500 +I0913 16:41:00.547108 140490712225536 logging_writer.py:48] [63000] global_step=63000, grad_norm=0.5, loss=0.873348 +I0913 16:41:00.551558 140514271229120 submission.py:307] 63000) loss = 0.873, grad_norm = 0.500 +I0913 16:48:08.268640 140490737403648 logging_writer.py:48] [63500] global_step=63500, grad_norm=0.5, loss=0.865903 +I0913 16:48:08.272922 140514271229120 submission.py:307] 63500) loss = 0.866, grad_norm = 0.500 +I0913 16:53:23.068590 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 16:54:23.463432 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 16:55:27.799176 140514271229120 spec.py:363] Evaluating on the test split. +I0913 16:55:28.873269 140514271229120 submission_runner.py:516] Time since start: 51283.94s, Step: 63810, {'train/accuracy': 0.7591677295918368, 'train/loss': 3.811367112763074, 'validation/accuracy': 0.59584, 'validation/loss': 4.0934078125, 'validation/num_examples': 50000, 'test/accuracy': 0.5409, 'test/loss': 2.397580859375, 'test/num_examples': 10000, 'score': 47999.9391438961, 'total_duration': 51283.937019109726, 'accumulated_submission_time': 47999.9391438961, 'accumulated_eval_time': 3172.961097240448, 'accumulated_logging_time': 9.259364604949951} +I0913 16:55:29.233761 140490804545280 logging_writer.py:48] [63810] accumulated_eval_time=3172.96, accumulated_logging_time=9.25936, accumulated_submission_time=47999.9, global_step=63810, preemption_count=0, score=47999.9, test/accuracy=0.5409, test/loss=2.39758, test/num_examples=10000, total_duration=51283.9, train/accuracy=0.759168, train/loss=3.81137, validation/accuracy=0.59584, validation/loss=4.09341, validation/num_examples=50000 +I0913 16:56:56.677914 140494889809664 logging_writer.py:48] [64000] global_step=64000, grad_norm=0.5, loss=0.94994 +I0913 16:56:56.682558 140514271229120 submission.py:307] 64000) loss = 0.950, grad_norm = 0.500 +I0913 17:03:00.326092 140490804545280 logging_writer.py:48] [64500] global_step=64500, grad_norm=0.5, loss=0.814626 +I0913 17:03:00.330140 140514271229120 submission.py:307] 64500) loss = 0.815, grad_norm = 0.500 +I0913 17:11:48.257926 140494889809664 logging_writer.py:48] [65000] global_step=65000, grad_norm=0.5, loss=1.02306 +I0913 17:11:48.261926 140514271229120 submission.py:307] 65000) loss = 1.023, grad_norm = 0.500 +I0913 17:16:01.345793 140490804545280 logging_writer.py:48] [65500] global_step=65500, grad_norm=0.5, loss=0.836719 +I0913 17:16:01.350038 140514271229120 submission.py:307] 65500) loss = 0.837, grad_norm = 0.500 +I0913 17:23:00.879281 140494889809664 logging_writer.py:48] [66000] global_step=66000, grad_norm=0.5, loss=1.11818 +I0913 17:23:00.883311 140514271229120 submission.py:307] 66000) loss = 1.118, grad_norm = 0.500 +I0913 17:28:47.791739 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 17:29:45.878079 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 17:30:50.258929 140514271229120 spec.py:363] Evaluating on the test split. +I0913 17:30:51.301173 140514271229120 submission_runner.py:516] Time since start: 53406.36s, Step: 66364, {'train/accuracy': 0.7644491390306123, 'train/loss': 3.9005092698700574, 'validation/accuracy': 0.6022, 'validation/loss': 3.884179375, 'validation/num_examples': 50000, 'test/accuracy': 0.5409, 'test/loss': 2.4544275390625, 'test/num_examples': 10000, 'score': 49994.2833840847, 'total_duration': 53406.36391091347, 'accumulated_submission_time': 49994.2833840847, 'accumulated_eval_time': 3296.469463825226, 'accumulated_logging_time': 9.629589080810547} +I0913 17:30:51.758772 140487004518144 logging_writer.py:48] [66364] accumulated_eval_time=3296.47, accumulated_logging_time=9.62959, accumulated_submission_time=49994.3, global_step=66364, preemption_count=0, score=49994.3, test/accuracy=0.5409, test/loss=2.45443, test/num_examples=10000, total_duration=53406.4, train/accuracy=0.764449, train/loss=3.90051, validation/accuracy=0.6022, validation/loss=3.88418, validation/num_examples=50000 +I0913 17:31:42.937378 140490687047424 logging_writer.py:48] [66500] global_step=66500, grad_norm=0.5, loss=0.840001 +I0913 17:31:42.942049 140514271229120 submission.py:307] 66500) loss = 0.840, grad_norm = 0.500 +I0913 17:37:54.355861 140487004518144 logging_writer.py:48] [67000] global_step=67000, grad_norm=0.5, loss=0.898243 +I0913 17:37:54.359773 140514271229120 submission.py:307] 67000) loss = 0.898, grad_norm = 0.500 +I0913 17:46:44.528836 140490687047424 logging_writer.py:48] [67500] global_step=67500, grad_norm=0.5, loss=0.956776 +I0913 17:46:44.532876 140514271229120 submission.py:307] 67500) loss = 0.957, grad_norm = 0.500 +I0913 17:50:52.928182 140487004518144 logging_writer.py:48] [68000] global_step=68000, grad_norm=0.5, loss=0.825535 +I0913 17:50:52.932488 140514271229120 submission.py:307] 68000) loss = 0.826, grad_norm = 0.500 +I0913 17:57:47.509945 140490687047424 logging_writer.py:48] [68500] global_step=68500, grad_norm=0.5, loss=0.840389 +I0913 17:57:47.514047 140514271229120 submission.py:307] 68500) loss = 0.840, grad_norm = 0.500 +I0913 18:04:09.749698 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 18:05:07.643313 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 18:06:12.172383 140514271229120 spec.py:363] Evaluating on the test split. +I0913 18:06:13.274280 140514271229120 submission_runner.py:516] Time since start: 55528.34s, Step: 68925, {'train/accuracy': 0.7666414221938775, 'train/loss': 3.916106554926658, 'validation/accuracy': 0.60528, 'validation/loss': 3.85621125, 'validation/num_examples': 50000, 'test/accuracy': 0.5375, 'test/loss': 2.504280078125, 'test/num_examples': 10000, 'score': 51988.392485141754, 'total_duration': 55528.33761072159, 'accumulated_submission_time': 51988.392485141754, 'accumulated_eval_time': 3419.99373793602, 'accumulated_logging_time': 10.09680700302124} +I0913 18:06:13.605709 140490678654720 logging_writer.py:48] [68925] accumulated_eval_time=3419.99, accumulated_logging_time=10.0968, accumulated_submission_time=51988.4, global_step=68925, preemption_count=0, score=51988.4, test/accuracy=0.5375, test/loss=2.50428, test/num_examples=10000, total_duration=55528.3, train/accuracy=0.766641, train/loss=3.91611, validation/accuracy=0.60528, validation/loss=3.85621, validation/num_examples=50000 +I0913 18:06:36.879869 140490745796352 logging_writer.py:48] [69000] global_step=69000, grad_norm=0.5, loss=0.871042 +I0913 18:06:36.883324 140514271229120 submission.py:307] 69000) loss = 0.871, grad_norm = 0.500 +I0913 18:12:20.615773 140490678654720 logging_writer.py:48] [69500] global_step=69500, grad_norm=0.5, loss=0.867041 +I0913 18:12:20.619791 140514271229120 submission.py:307] 69500) loss = 0.867, grad_norm = 0.500 +I0913 18:21:08.886105 140490745796352 logging_writer.py:48] [70000] global_step=70000, grad_norm=0.5, loss=0.890044 +I0913 18:21:08.890021 140514271229120 submission.py:307] 70000) loss = 0.890, grad_norm = 0.500 +I0913 18:25:20.476307 140490678654720 logging_writer.py:48] [70500] global_step=70500, grad_norm=0.5, loss=0.902929 +I0913 18:25:20.480630 140514271229120 submission.py:307] 70500) loss = 0.903, grad_norm = 0.500 +I0913 18:32:12.865567 140490745796352 logging_writer.py:48] [71000] global_step=71000, grad_norm=0.5, loss=0.830665 +I0913 18:32:12.869651 140514271229120 submission.py:307] 71000) loss = 0.831, grad_norm = 0.500 +I0913 18:38:59.810818 140490678654720 logging_writer.py:48] [71500] global_step=71500, grad_norm=0.5, loss=0.838571 +I0913 18:38:59.815694 140514271229120 submission.py:307] 71500) loss = 0.839, grad_norm = 0.500 +I0913 18:39:32.936943 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 18:40:29.039332 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 18:41:34.122836 140514271229120 spec.py:363] Evaluating on the test split. +I0913 18:41:35.305552 140514271229120 submission_runner.py:516] Time since start: 57650.37s, Step: 71567, {'train/accuracy': 0.7959582270408163, 'train/loss': 3.3684337382413903, 'validation/accuracy': 0.61068, 'validation/loss': 3.876725625, 'validation/num_examples': 50000, 'test/accuracy': 0.5439, 'test/loss': 2.482401171875, 'test/num_examples': 10000, 'score': 53983.56141471863, 'total_duration': 57650.36901497841, 'accumulated_submission_time': 53983.56141471863, 'accumulated_eval_time': 3542.36199259758, 'accumulated_logging_time': 10.43744444847107} +I0913 18:41:35.734674 140490779367168 logging_writer.py:48] [71567] accumulated_eval_time=3542.36, accumulated_logging_time=10.4374, accumulated_submission_time=53983.6, global_step=71567, preemption_count=0, score=53983.6, test/accuracy=0.5439, test/loss=2.4824, test/num_examples=10000, total_duration=57650.4, train/accuracy=0.795958, train/loss=3.36843, validation/accuracy=0.61068, validation/loss=3.87673, validation/num_examples=50000 +I0913 18:46:24.094185 140490796152576 logging_writer.py:48] [72000] global_step=72000, grad_norm=0.5, loss=0.780881 +I0913 18:46:24.098392 140514271229120 submission.py:307] 72000) loss = 0.781, grad_norm = 0.500 +I0913 18:55:12.874040 140490779367168 logging_writer.py:48] [72500] global_step=72500, grad_norm=0.5, loss=0.915551 +I0913 18:55:12.878271 140514271229120 submission.py:307] 72500) loss = 0.916, grad_norm = 0.500 +I0913 18:59:32.097681 140490796152576 logging_writer.py:48] [73000] global_step=73000, grad_norm=0.5, loss=0.891877 +I0913 18:59:32.102020 140514271229120 submission.py:307] 73000) loss = 0.892, grad_norm = 0.500 +I0913 19:06:25.880032 140490779367168 logging_writer.py:48] [73500] global_step=73500, grad_norm=0.5, loss=0.863688 +I0913 19:06:25.883996 140514271229120 submission.py:307] 73500) loss = 0.864, grad_norm = 0.500 +I0913 19:13:06.137758 140490796152576 logging_writer.py:48] [74000] global_step=74000, grad_norm=0.5, loss=0.772857 +I0913 19:13:06.178045 140514271229120 submission.py:307] 74000) loss = 0.773, grad_norm = 0.500 +I0913 19:14:52.785839 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 19:15:55.748608 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 19:17:00.517820 140514271229120 spec.py:363] Evaluating on the test split. +I0913 19:17:01.489307 140514271229120 submission_runner.py:516] Time since start: 59776.55s, Step: 74216, {'train/accuracy': 0.7823262117346939, 'train/loss': 3.757672368263712, 'validation/accuracy': 0.60554, 'validation/loss': 4.1128596875, 'validation/num_examples': 50000, 'test/accuracy': 0.5405, 'test/loss': 2.5159078125, 'test/num_examples': 10000, 'score': 55976.11931800842, 'total_duration': 59776.55257821083, 'accumulated_submission_time': 55976.11931800842, 'accumulated_eval_time': 3671.0648312568665, 'accumulated_logging_time': 10.87571096420288} +I0913 19:17:01.873132 140490745796352 logging_writer.py:48] [74216] accumulated_eval_time=3671.06, accumulated_logging_time=10.8757, accumulated_submission_time=55976.1, global_step=74216, preemption_count=0, score=55976.1, test/accuracy=0.5405, test/loss=2.51591, test/num_examples=10000, total_duration=59776.6, train/accuracy=0.782326, train/loss=3.75767, validation/accuracy=0.60554, validation/loss=4.11286, validation/num_examples=50000 +I0913 19:20:15.898080 140494889809664 logging_writer.py:48] [74500] global_step=74500, grad_norm=0.5, loss=0.838182 +I0913 19:20:15.902332 140514271229120 submission.py:307] 74500) loss = 0.838, grad_norm = 0.500 +I0913 19:28:59.596987 140490745796352 logging_writer.py:48] [75000] global_step=75000, grad_norm=0.5, loss=0.895016 +I0913 19:28:59.601127 140514271229120 submission.py:307] 75000) loss = 0.895, grad_norm = 0.500 +I0913 19:33:22.162760 140494889809664 logging_writer.py:48] [75500] global_step=75500, grad_norm=0.5, loss=0.774945 +I0913 19:33:22.167322 140514271229120 submission.py:307] 75500) loss = 0.775, grad_norm = 0.500 +I0913 19:40:03.455350 140490745796352 logging_writer.py:48] [76000] global_step=76000, grad_norm=0.5, loss=0.810245 +I0913 19:40:03.459419 140514271229120 submission.py:307] 76000) loss = 0.810, grad_norm = 0.500 +I0913 19:46:52.769191 140494889809664 logging_writer.py:48] [76500] global_step=76500, grad_norm=0.5, loss=0.763745 +I0913 19:46:52.775023 140514271229120 submission.py:307] 76500) loss = 0.764, grad_norm = 0.500 +I0913 19:50:23.607536 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 19:51:31.547399 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 19:52:38.598563 140514271229120 spec.py:363] Evaluating on the test split. +I0913 19:52:39.517490 140514271229120 submission_runner.py:516] Time since start: 61914.58s, Step: 76891, {'train/accuracy': 0.7877471301020408, 'train/loss': 3.733782709861288, 'validation/accuracy': 0.60232, 'validation/loss': 4.02851125, 'validation/num_examples': 50000, 'test/accuracy': 0.5366, 'test/loss': 2.53688671875, 'test/num_examples': 10000, 'score': 57973.79527235031, 'total_duration': 61914.58137059212, 'accumulated_submission_time': 57973.79527235031, 'accumulated_eval_time': 3806.97487950325, 'accumulated_logging_time': 11.273123502731323} +I0913 19:52:39.842065 140490821330688 logging_writer.py:48] [76891] accumulated_eval_time=3806.97, accumulated_logging_time=11.2731, accumulated_submission_time=57973.8, global_step=76891, preemption_count=0, score=57973.8, test/accuracy=0.5366, test/loss=2.53689, test/num_examples=10000, total_duration=61914.6, train/accuracy=0.787747, train/loss=3.73378, validation/accuracy=0.60232, validation/loss=4.02851, validation/num_examples=50000 +I0913 19:53:23.410523 140490720618240 logging_writer.py:48] [77000] global_step=77000, grad_norm=0.5, loss=0.854225 +I0913 19:53:23.413723 140514271229120 submission.py:307] 77000) loss = 0.854, grad_norm = 0.500 +I0913 20:02:10.863173 140490821330688 logging_writer.py:48] [77500] global_step=77500, grad_norm=0.5, loss=0.86312 +I0913 20:02:10.867010 140514271229120 submission.py:307] 77500) loss = 0.863, grad_norm = 0.500 +I0913 20:06:45.732349 140490720618240 logging_writer.py:48] [78000] global_step=78000, grad_norm=0.5, loss=0.8193 +I0913 20:06:45.738946 140514271229120 submission.py:307] 78000) loss = 0.819, grad_norm = 0.500 +I0913 20:13:34.235572 140490821330688 logging_writer.py:48] [78500] global_step=78500, grad_norm=0.5, loss=0.801849 +I0913 20:13:34.239620 140514271229120 submission.py:307] 78500) loss = 0.802, grad_norm = 0.500 +I0913 20:20:19.087476 140490720618240 logging_writer.py:48] [79000] global_step=79000, grad_norm=0.5, loss=0.791576 +I0913 20:20:19.094136 140514271229120 submission.py:307] 79000) loss = 0.792, grad_norm = 0.500 +I0913 20:25:06.567532 140490821330688 logging_writer.py:48] [79500] global_step=79500, grad_norm=0.5, loss=0.811931 +I0913 20:25:06.571855 140514271229120 submission.py:307] 79500) loss = 0.812, grad_norm = 0.500 +I0913 20:25:58.167537 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 20:27:00.596070 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 20:28:21.095267 140514271229120 spec.py:363] Evaluating on the test split. +I0913 20:28:22.100671 140514271229120 submission_runner.py:516] Time since start: 64057.16s, Step: 79571, {'train/accuracy': 0.7647082270408163, 'train/loss': 4.528148417570153, 'validation/accuracy': 0.59836, 'validation/loss': 4.343521875, 'validation/num_examples': 50000, 'test/accuracy': 0.542, 'test/loss': 2.5561978515625, 'test/num_examples': 10000, 'score': 59967.32413458824, 'total_duration': 64057.164492845535, 'accumulated_submission_time': 59967.32413458824, 'accumulated_eval_time': 3950.907996416092, 'accumulated_logging_time': 11.607618570327759} +I0913 20:28:22.435104 140490720618240 logging_writer.py:48] [79571] accumulated_eval_time=3950.91, accumulated_logging_time=11.6076, accumulated_submission_time=59967.3, global_step=79571, preemption_count=0, score=59967.3, test/accuracy=0.542, test/loss=2.5562, test/num_examples=10000, total_duration=64057.2, train/accuracy=0.764708, train/loss=4.52815, validation/accuracy=0.59836, validation/loss=4.34352, validation/num_examples=50000 +I0913 20:34:51.908214 140490737403648 logging_writer.py:48] [80000] global_step=80000, grad_norm=0.5, loss=0.917458 +I0913 20:34:51.912270 140514271229120 submission.py:307] 80000) loss = 0.917, grad_norm = 0.500 +I0913 20:39:40.877591 140490720618240 logging_writer.py:48] [80500] global_step=80500, grad_norm=0.5, loss=0.81624 +I0913 20:39:40.882082 140514271229120 submission.py:307] 80500) loss = 0.816, grad_norm = 0.500 +I0913 20:46:17.941640 140490737403648 logging_writer.py:48] [81000] global_step=81000, grad_norm=0.5, loss=0.880406 +I0913 20:46:17.947255 140514271229120 submission.py:307] 81000) loss = 0.880, grad_norm = 0.500 +I0913 20:53:11.065315 140490720618240 logging_writer.py:48] [81500] global_step=81500, grad_norm=0.5, loss=0.772928 +I0913 20:53:11.069930 140514271229120 submission.py:307] 81500) loss = 0.773, grad_norm = 0.500 +I0913 20:57:58.795092 140490737403648 logging_writer.py:48] [82000] global_step=82000, grad_norm=0.5, loss=0.831165 +I0913 20:57:58.802070 140514271229120 submission.py:307] 82000) loss = 0.831, grad_norm = 0.500 +I0913 21:01:44.141921 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 21:02:34.282312 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 21:03:51.044844 140514271229120 spec.py:363] Evaluating on the test split. +I0913 21:03:52.045162 140514271229120 submission_runner.py:516] Time since start: 66187.11s, Step: 82235, {'train/accuracy': 0.7634526466836735, 'train/loss': 4.757924916792889, 'validation/accuracy': 0.60174, 'validation/loss': 4.3246, 'validation/num_examples': 50000, 'test/accuracy': 0.5413, 'test/loss': 2.5651658203125, 'test/num_examples': 10000, 'score': 61964.11759972572, 'total_duration': 66187.10884904861, 'accumulated_submission_time': 61964.11759972572, 'accumulated_eval_time': 4078.8110699653625, 'accumulated_logging_time': 11.951503276824951} +I0913 21:03:52.445187 140490745796352 logging_writer.py:48] [82235] accumulated_eval_time=4078.81, accumulated_logging_time=11.9515, accumulated_submission_time=61964.1, global_step=82235, preemption_count=0, score=61964.1, test/accuracy=0.5413, test/loss=2.56517, test/num_examples=10000, total_duration=66187.1, train/accuracy=0.763453, train/loss=4.75792, validation/accuracy=0.60174, validation/loss=4.3246, validation/num_examples=50000 +I0913 21:07:32.418858 140490804545280 logging_writer.py:48] [82500] global_step=82500, grad_norm=0.5, loss=0.734869 +I0913 21:07:32.422840 140514271229120 submission.py:307] 82500) loss = 0.735, grad_norm = 0.500 +I0913 21:12:26.924816 140490745796352 logging_writer.py:48] [83000] global_step=83000, grad_norm=0.5, loss=0.821738 +I0913 21:12:26.929152 140514271229120 submission.py:307] 83000) loss = 0.822, grad_norm = 0.500 +I0913 21:19:18.790069 140490804545280 logging_writer.py:48] [83500] global_step=83500, grad_norm=0.5, loss=0.817375 +I0913 21:19:18.794127 140514271229120 submission.py:307] 83500) loss = 0.817, grad_norm = 0.500 +I0913 21:26:00.908625 140490745796352 logging_writer.py:48] [84000] global_step=84000, grad_norm=0.5, loss=0.795605 +I0913 21:26:00.913128 140514271229120 submission.py:307] 84000) loss = 0.796, grad_norm = 0.500 +I0913 21:30:49.552142 140490804545280 logging_writer.py:48] [84500] global_step=84500, grad_norm=0.5, loss=0.871115 +I0913 21:30:49.556916 140514271229120 submission.py:307] 84500) loss = 0.871, grad_norm = 0.500 +I0913 21:37:09.150113 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 21:37:58.841524 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 21:39:37.387074 140514271229120 spec.py:363] Evaluating on the test split. +I0913 21:39:38.410038 140514271229120 submission_runner.py:516] Time since start: 68333.47s, Step: 84895, {'train/accuracy': 0.7957589285714286, 'train/loss': 3.9244023537149233, 'validation/accuracy': 0.60644, 'validation/loss': 4.156065625, 'validation/num_examples': 50000, 'test/accuracy': 0.5449, 'test/loss': 2.5498849609375, 'test/num_examples': 10000, 'score': 63957.24264502525, 'total_duration': 68333.47369885445, 'accumulated_submission_time': 63957.24264502525, 'accumulated_eval_time': 4228.070791721344, 'accumulated_logging_time': 12.361259937286377} +I0913 21:39:38.724775 140490821330688 logging_writer.py:48] [84895] accumulated_eval_time=4228.07, accumulated_logging_time=12.3613, accumulated_submission_time=63957.2, global_step=84895, preemption_count=0, score=63957.2, test/accuracy=0.5449, test/loss=2.54988, test/num_examples=10000, total_duration=68333.5, train/accuracy=0.795759, train/loss=3.9244, validation/accuracy=0.60644, validation/loss=4.15607, validation/num_examples=50000 +I0913 21:40:25.101240 140490720618240 logging_writer.py:48] [85000] global_step=85000, grad_norm=0.5, loss=0.894404 +I0913 21:40:25.104959 140514271229120 submission.py:307] 85000) loss = 0.894, grad_norm = 0.500 +I0913 21:45:28.792019 140490821330688 logging_writer.py:48] [85500] global_step=85500, grad_norm=0.5, loss=0.745501 +I0913 21:45:28.796593 140514271229120 submission.py:307] 85500) loss = 0.746, grad_norm = 0.500 +I0913 21:52:44.987560 140490720618240 logging_writer.py:48] [86000] global_step=86000, grad_norm=0.5, loss=0.850102 +I0913 21:52:44.991847 140514271229120 submission.py:307] 86000) loss = 0.850, grad_norm = 0.500 +I0913 21:59:13.237406 140490821330688 logging_writer.py:48] [86500] global_step=86500, grad_norm=0.5, loss=0.758838 +I0913 21:59:13.242027 140514271229120 submission.py:307] 86500) loss = 0.759, grad_norm = 0.500 +I0913 22:04:06.094130 140490720618240 logging_writer.py:48] [87000] global_step=87000, grad_norm=0.5, loss=0.732425 +I0913 22:04:06.098297 140514271229120 submission.py:307] 87000) loss = 0.732, grad_norm = 0.500 +I0913 22:12:33.475218 140490821330688 logging_writer.py:48] [87500] global_step=87500, grad_norm=0.5, loss=0.76094 +I0913 22:12:33.479147 140514271229120 submission.py:307] 87500) loss = 0.761, grad_norm = 0.500 +I0913 22:12:56.365376 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 22:13:36.525600 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 22:14:40.920012 140514271229120 spec.py:363] Evaluating on the test split. +I0913 22:14:41.954189 140514271229120 submission_runner.py:516] Time since start: 70437.02s, Step: 87519, {'train/accuracy': 0.7683952487244898, 'train/loss': 4.895434009785554, 'validation/accuracy': 0.60802, 'validation/loss': 4.351689375, 'validation/num_examples': 50000, 'test/accuracy': 0.5401, 'test/loss': 2.64440703125, 'test/num_examples': 10000, 'score': 65951.39333939552, 'total_duration': 70437.01794242859, 'accumulated_submission_time': 65951.39333939552, 'accumulated_eval_time': 4333.659548521042, 'accumulated_logging_time': 12.685445070266724} +I0913 22:14:42.267363 140490829723392 logging_writer.py:48] [87519] accumulated_eval_time=4333.66, accumulated_logging_time=12.6854, accumulated_submission_time=65951.4, global_step=87519, preemption_count=0, score=65951.4, test/accuracy=0.5401, test/loss=2.64441, test/num_examples=10000, total_duration=70437, train/accuracy=0.768395, train/loss=4.89543, validation/accuracy=0.60802, validation/loss=4.35169, validation/num_examples=50000 +I0913 22:19:30.322765 140490703832832 logging_writer.py:48] [88000] global_step=88000, grad_norm=0.5, loss=0.702781 +I0913 22:19:30.327003 140514271229120 submission.py:307] 88000) loss = 0.703, grad_norm = 0.500 +I0913 22:26:45.894673 140490829723392 logging_writer.py:48] [88500] global_step=88500, grad_norm=0.5, loss=0.822346 +I0913 22:26:45.898950 140514271229120 submission.py:307] 88500) loss = 0.822, grad_norm = 0.500 +I0913 22:33:34.418975 140490703832832 logging_writer.py:48] [89000] global_step=89000, grad_norm=0.5, loss=0.686462 +I0913 22:33:34.423540 140514271229120 submission.py:307] 89000) loss = 0.686, grad_norm = 0.500 +I0913 22:38:30.200774 140490829723392 logging_writer.py:48] [89500] global_step=89500, grad_norm=0.5, loss=0.727546 +I0913 22:38:30.205158 140514271229120 submission.py:307] 89500) loss = 0.728, grad_norm = 0.500 +I0913 22:47:15.540215 140490703832832 logging_writer.py:48] [90000] global_step=90000, grad_norm=0.5, loss=0.794669 +I0913 22:47:15.544305 140514271229120 submission.py:307] 90000) loss = 0.795, grad_norm = 0.500 +I0913 22:48:06.371295 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 22:49:01.406388 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 22:50:05.935353 140514271229120 spec.py:363] Evaluating on the test split. +I0913 22:50:06.984654 140514271229120 submission_runner.py:516] Time since start: 72562.05s, Step: 90073, {'train/accuracy': 0.7809709821428571, 'train/loss': 4.373063690808355, 'validation/accuracy': 0.61414, 'validation/loss': 3.89102, 'validation/num_examples': 50000, 'test/accuracy': 0.5335, 'test/loss': 2.604086328125, 'test/num_examples': 10000, 'score': 67952.03266215324, 'total_duration': 72562.04839730263, 'accumulated_submission_time': 67952.03266215324, 'accumulated_eval_time': 4454.272779941559, 'accumulated_logging_time': 13.008050680160522} +I0913 22:50:07.372201 140490821330688 logging_writer.py:48] [90073] accumulated_eval_time=4454.27, accumulated_logging_time=13.0081, accumulated_submission_time=67952, global_step=90073, preemption_count=0, score=67952, test/accuracy=0.5335, test/loss=2.60409, test/num_examples=10000, total_duration=72562, train/accuracy=0.780971, train/loss=4.37306, validation/accuracy=0.61414, validation/loss=3.89102, validation/num_examples=50000 +I0913 22:54:16.358023 140490695440128 logging_writer.py:48] [90500] global_step=90500, grad_norm=0.5, loss=0.745627 +I0913 22:54:16.362180 140514271229120 submission.py:307] 90500) loss = 0.746, grad_norm = 0.500 +I0913 23:01:40.843982 140490821330688 logging_writer.py:48] [91000] global_step=91000, grad_norm=0.5, loss=0.778148 +I0913 23:01:40.847941 140514271229120 submission.py:307] 91000) loss = 0.778, grad_norm = 0.500 +I0913 23:08:27.301154 140490695440128 logging_writer.py:48] [91500] global_step=91500, grad_norm=0.5, loss=0.698589 +I0913 23:08:27.305855 140514271229120 submission.py:307] 91500) loss = 0.699, grad_norm = 0.500 +I0913 23:13:14.950436 140490821330688 logging_writer.py:48] [92000] global_step=92000, grad_norm=0.5, loss=0.704468 +I0913 23:13:14.954623 140514271229120 submission.py:307] 92000) loss = 0.704, grad_norm = 0.500 +I0913 23:21:50.713141 140490695440128 logging_writer.py:48] [92500] global_step=92500, grad_norm=0.5, loss=0.682563 +I0913 23:21:50.717218 140514271229120 submission.py:307] 92500) loss = 0.683, grad_norm = 0.500 +I0913 23:23:24.560566 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 23:24:24.470755 140514271229120 spec.py:346] Evaluating on the validation split. +I0913 23:25:28.896778 140514271229120 spec.py:363] Evaluating on the test split. +I0913 23:25:29.893584 140514271229120 submission_runner.py:516] Time since start: 74684.96s, Step: 92626, {'train/accuracy': 0.7737364477040817, 'train/loss': 4.867963518415179, 'validation/accuracy': 0.59746, 'validation/loss': 4.616288125, 'validation/num_examples': 50000, 'test/accuracy': 0.538, 'test/loss': 2.6560337890625, 'test/num_examples': 10000, 'score': 69945.19483971596, 'total_duration': 74684.95734524727, 'accumulated_submission_time': 69945.19483971596, 'accumulated_eval_time': 4579.605948209763, 'accumulated_logging_time': 13.406023025512695} +I0913 23:25:30.268054 140490745796352 logging_writer.py:48] [92626] accumulated_eval_time=4579.61, accumulated_logging_time=13.406, accumulated_submission_time=69945.2, global_step=92626, preemption_count=0, score=69945.2, test/accuracy=0.538, test/loss=2.65603, test/num_examples=10000, total_duration=74685, train/accuracy=0.773736, train/loss=4.86796, validation/accuracy=0.59746, validation/loss=4.61629, validation/num_examples=50000 +I0913 23:28:55.449687 140490762581760 logging_writer.py:48] [93000] global_step=93000, grad_norm=0.5, loss=0.733097 +I0913 23:28:55.454209 140514271229120 submission.py:307] 93000) loss = 0.733, grad_norm = 0.500 +I0913 23:36:04.400618 140490745796352 logging_writer.py:48] [93500] global_step=93500, grad_norm=0.5, loss=0.74871 +I0913 23:36:04.404711 140514271229120 submission.py:307] 93500) loss = 0.749, grad_norm = 0.500 +I0913 23:43:04.904014 140490762581760 logging_writer.py:48] [94000] global_step=94000, grad_norm=0.5, loss=0.694402 +I0913 23:43:04.908900 140514271229120 submission.py:307] 94000) loss = 0.694, grad_norm = 0.500 +I0913 23:47:51.597228 140490745796352 logging_writer.py:48] [94500] global_step=94500, grad_norm=0.5, loss=0.753358 +I0913 23:47:51.602289 140514271229120 submission.py:307] 94500) loss = 0.753, grad_norm = 0.500 +I0913 23:56:24.143630 140490762581760 logging_writer.py:48] [95000] global_step=95000, grad_norm=0.5, loss=0.793264 +I0913 23:56:24.148485 140514271229120 submission.py:307] 95000) loss = 0.793, grad_norm = 0.500 +I0913 23:58:48.056413 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0913 23:59:47.454836 140514271229120 spec.py:346] Evaluating on the validation split. +I0914 00:00:52.025453 140514271229120 spec.py:363] Evaluating on the test split. +I0914 00:00:53.051068 140514271229120 submission_runner.py:516] Time since start: 76808.11s, Step: 95245, {'train/accuracy': 0.7448182397959183, 'train/loss': 5.682906793088329, 'validation/accuracy': 0.61956, 'validation/loss': 3.79448125, 'validation/num_examples': 50000, 'test/accuracy': 0.546, 'test/loss': 2.6514984375, 'test/num_examples': 10000, 'score': 71939.20494151115, 'total_duration': 76808.11458563805, 'accumulated_submission_time': 71939.20494151115, 'accumulated_eval_time': 4704.600276708603, 'accumulated_logging_time': 13.790885210037231} +I0914 00:00:53.467676 140490829723392 logging_writer.py:48] [95245] accumulated_eval_time=4704.6, accumulated_logging_time=13.7909, accumulated_submission_time=71939.2, global_step=95245, preemption_count=0, score=71939.2, test/accuracy=0.546, test/loss=2.6515, test/num_examples=10000, total_duration=76808.1, train/accuracy=0.744818, train/loss=5.68291, validation/accuracy=0.61956, validation/loss=3.79448, validation/num_examples=50000 +I0914 00:03:12.974250 140490796152576 logging_writer.py:48] [95500] global_step=95500, grad_norm=0.5, loss=0.646175 +I0914 00:03:12.978471 140514271229120 submission.py:307] 95500) loss = 0.646, grad_norm = 0.500 +I0914 00:10:32.495005 140490829723392 logging_writer.py:48] [96000] global_step=96000, grad_norm=0.5, loss=0.589081 +I0914 00:10:32.499438 140514271229120 submission.py:307] 96000) loss = 0.589, grad_norm = 0.500 +I0914 00:17:43.814709 140490796152576 logging_writer.py:48] [96500] global_step=96500, grad_norm=0.5, loss=0.671435 +I0914 00:17:43.825751 140514271229120 submission.py:307] 96500) loss = 0.671, grad_norm = 0.500 +I0914 00:22:33.129026 140490829723392 logging_writer.py:48] [97000] global_step=97000, grad_norm=0.5, loss=0.66773 +I0914 00:22:33.135539 140514271229120 submission.py:307] 97000) loss = 0.668, grad_norm = 0.500 +I0914 00:31:03.270990 140490796152576 logging_writer.py:48] [97500] global_step=97500, grad_norm=0.5, loss=0.744652 +I0914 00:31:03.275207 140514271229120 submission.py:307] 97500) loss = 0.745, grad_norm = 0.500 +I0914 00:34:10.160979 140514271229120 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0914 00:35:08.681772 140514271229120 spec.py:346] Evaluating on the validation split. +I0914 00:36:13.022355 140514271229120 spec.py:363] Evaluating on the test split. +I0914 00:36:14.039400 140514271229120 submission_runner.py:516] Time since start: 78929.10s, Step: 97839, {'train/accuracy': 0.7731784119897959, 'train/loss': 4.878094731544961, 'validation/accuracy': 0.61604, 'validation/loss': 4.014446875, 'validation/num_examples': 50000, 'test/accuracy': 0.5366, 'test/loss': 2.770053125, 'test/num_examples': 10000, 'score': 73931.97438669205, 'total_duration': 78929.10322093964, 'accumulated_submission_time': 73931.97438669205, 'accumulated_eval_time': 4828.478740930557, 'accumulated_logging_time': 14.217012643814087} +I0914 00:36:14.377723 140490796152576 logging_writer.py:48] [97839] accumulated_eval_time=4828.48, accumulated_logging_time=14.217, accumulated_submission_time=73932, global_step=97839, preemption_count=0, score=73932, test/accuracy=0.5366, test/loss=2.77005, test/num_examples=10000, total_duration=78929.1, train/accuracy=0.773178, train/loss=4.87809, validation/accuracy=0.61604, validation/loss=4.01445, validation/num_examples=50000 +I0914 00:37:28.027981 140490812937984 logging_writer.py:48] [98000] global_step=98000, grad_norm=0.5, loss=0.683372 +I0914 00:37:28.031917 140514271229120 submission.py:307] 98000) loss = 0.683, grad_norm = 0.500 +I0914 00:44:47.624146 140490796152576 logging_writer.py:48] [98500] global_step=98500, grad_norm=0.5, loss=0.707994 +I0914 00:44:47.628294 140514271229120 submission.py:307] 98500) loss = 0.708, grad_norm = 0.500 +I0914 00:51:59.330952 140490812937984 logging_writer.py:48] [99000] global_step=99000, grad_norm=0.5, loss=0.633483 +I0914 00:51:59.335550 140514271229120 submission.py:307] 99000) loss = 0.633, grad_norm = 0.500 +I0914 00:56:45.834876 140490796152576 logging_writer.py:48] [99500] global_step=99500, grad_norm=0.5, loss=0.711325 +I0914 00:56:45.839728 140514271229120 submission.py:307] 99500) loss = 0.711, grad_norm = 0.500 +I0914 01:05:14.453270 140490812937984 logging_writer.py:48] [100000] global_step=100000, grad_norm=0.5, loss=0.70608 +I0914 01:05:14.457441 140514271229120 submission.py:307] 100000) loss = 0.706, grad_norm = 0.500 +I0914 01:09:32.741427 140490796152576 logging_writer.py:48] [100480] global_step=100480, preemption_count=0, score=75927.6 +I0914 01:09:39.043331 140514271229120 submission_runner.py:857] Final imagenet_resnet score: 75927.63577508926 diff --git a/logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..8e13e8c8 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,39 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/accuracy,test/loss,test/num_examples,total_duration,train/accuracy,train/loss,validation/accuracy,validation/loss,validation/num_examples +188.0332458019257,0.0,117.69383144378662,1,0,117.69383144378662,0.0011,6.913284375,10000,306.40399718284607,0.0010363520408163,6.913094034000319,0.00048,6.91549375,50000 +309.0745980739593,0.031135082244873,2108.175397634506,3229,0,2108.175397634506,0.1165,4.81738046875,10000,2424.668477535248,0.1621293048469387,5.140706276407047,0.15864,4.9367896875,50000 +447.5696070194245,0.3852415084838867,4105.261125326157,5985,0,4105.261125326157,0.2437,3.977594921875,10000,4564.438182592392,0.3155691964285714,4.7227113684829405,0.3118,4.254304375,50000 +588.3147366046906,0.769068717956543,6103.339204072952,8666,0,6103.339204072952,0.3598,3.02988828125,10000,6707.132169008255,0.4215760522959184,4.734850513691804,0.4368,3.561740625,50000 +706.2089529037476,1.0964832305908203,8096.436839818954,11342,0,8096.436839818954,0.419,2.7275841796875,10000,8822.90674829483,0.519172512755102,3.8327312858737246,0.49202,3.432515625,50000 +829.4630072116852,1.425502061843872,10089.947731256483,14006,0,10089.947731256483,0.4528,2.601360546875,10000,10943.603086471558,0.5588329081632653,3.752557248485332,0.52142,3.3949671875,50000 +958.7037978172302,1.7975919246673584,12082.892327547072,16644,0,12082.892327547072,0.4807,2.3738681640625,10000,13070.141052007675,0.5981146364795918,3.628994377291933,0.54502,3.5477440625,50000 +1083.2835838794708,2.191770076751709,14081.901285886765,19287,0,14081.901285886765,0.4909,2.3680232421875,10000,15199.4295668602,0.5989516900510204,3.800162179129464,0.55098,3.3991665625,50000 +1209.900592803955,2.501425504684448,16076.210381507874,21970,0,16076.210381507874,0.4916,2.3627771484375,10000,17326.093100070953,0.5838647959183674,4.3787888507453765,0.56536,3.209695625,50000 +1352.7498168945312,2.8937878608703613,18070.13053560257,24669,0,18070.13053560257,0.5159,2.26638359375,10000,19468.0190050602,0.640625,3.562229078643176,0.56716,3.4612678125,50000 +1487.866794347763,3.247710704803467,20065.60669398308,27331,0,20065.60669398308,0.5157,2.247359765625,10000,21602.546391010284,0.6507095025510204,3.3582365074936225,0.5733,3.3512609375,50000 +1599.0447170734406,3.656982898712158,22061.40389227867,29969,0,22061.40389227867,0.5312,2.207117578125,10000,23713.798048257828,0.6632852359693877,3.6978236607142856,0.59528,3.07289625,50000 +1719.7836360931396,4.115094661712647,24056.594072580338,32527,0,24056.594072580338,0.5319,2.2375390625,10000,25833.816573143005,0.6575454400510204,4.162380296356824,0.58764,3.505524375,50000 +1841.544192790985,4.55816388130188,26049.596282482147,35088,0,26049.596282482147,0.5413,2.1987033203125,10000,27952.762268304825,0.6999362244897959,3.3552706971460458,0.59926,3.23747375,50000 +1965.5194764137268,5.167999982833862,28042.45645093918,37681,0,28042.45645093918,0.527,2.26468828125,10000,30073.98717403412,0.7167171556122449,3.406367710658482,0.60236,3.4714803125,50000 +2095.101897716522,5.541431188583374,30035.08023881912,40351,0,30035.08023881912,0.5265,2.308673828125,10000,32200.80206418037,0.7042012117346939,3.533531033262914,0.58318,3.826075,50000 +2225.492896080017,5.898090839385986,32028.396780967712,42975,0,32028.396780967712,0.5359,2.2669248046875,10000,34328.45838928223,0.7175143494897959,3.674822748923788,0.60794,3.3489609375,50000 +2350.798717737198,6.316791296005249,34023.23478484154,45639,0,34023.23478484154,0.5427,2.2691771484375,10000,36454.61201810837,0.7144650829081632,3.772730924645249,0.60726,3.3839653125,50000 +2483.848254919052,6.711942195892334,36016.49505734444,48310,0,36016.49505734444,0.5378,2.2942724609375,10000,38586.453929424286,0.7372249681122449,3.4263698032924106,0.59002,3.86083,50000 +2615.5865807533264,7.114745140075684,38012.98178291321,50983,0,38012.98178291321,0.5423,2.321203125,10000,40719.979044914246,0.7030253507653061,4.454295878507653,0.60278,3.635655625,50000 +2751.5167932510376,7.502183437347412,40020.47555828095,53643,0,40020.47555828095,0.5485,2.3113056640625,10000,42867.38005900383,0.7354512117346939,3.9732946279097576,0.60552,3.769000625,50000 +2847.636313676834,7.968269109725952,42013.84645605087,56265,0,42013.84645605087,0.5361,2.37835546875,10000,44960.60993313789,0.7109375,4.522355994399713,0.60638,3.6884934375,50000 +2950.9123871326447,8.387396335601807,44008.71176171303,58754,0,44008.71176171303,0.5422,2.350933203125,10000,47062.44623923302,0.7159996811224489,4.573967135682398,0.5886,4.122043125,50000 +3047.156206846237,8.810739994049072,46006.77871727944,61268,0,46006.77871727944,0.5413,2.3884078125,10000,49160.60372567177,0.7661830357142857,3.472351697026467,0.59942,4.078488125,50000 +3172.961097240448,9.259364604949951,47999.9391438961,63810,0,47999.9391438961,0.5409,2.397580859375,10000,51283.937019109726,0.7591677295918368,3.811367112763074,0.59584,4.0934078125,50000 +3296.469463825226,9.629589080810549,49994.2833840847,66364,0,49994.2833840847,0.5409,2.4544275390625,10000,53406.36391091347,0.7644491390306123,3.9005092698700574,0.6022,3.884179375,50000 +3419.99373793602,10.09680700302124,51988.39248514176,68925,0,51988.39248514176,0.5375,2.504280078125,10000,55528.33761072159,0.7666414221938775,3.916106554926658,0.60528,3.85621125,50000 +3542.36199259758,10.43744444847107,53983.56141471863,71567,0,53983.56141471863,0.5439,2.482401171875,10000,57650.36901497841,0.7959582270408163,3.3684337382413903,0.61068,3.876725625,50000 +3671.0648312568665,10.87571096420288,55976.11931800842,74216,0,55976.11931800842,0.5405,2.5159078125,10000,59776.55257821083,0.7823262117346939,3.757672368263712,0.60554,4.1128596875,50000 +3806.97487950325,11.273123502731323,57973.79527235031,76891,0,57973.79527235031,0.5366,2.53688671875,10000,61914.58137059212,0.7877471301020408,3.733782709861288,0.60232,4.02851125,50000 +3950.907996416092,11.60761857032776,59967.32413458824,79571,0,59967.32413458824,0.542,2.5561978515625,10000,64057.164492845535,0.7647082270408163,4.528148417570153,0.59836,4.343521875,50000 +4078.8110699653625,11.951503276824951,61964.11759972572,82235,0,61964.11759972572,0.5413,2.5651658203125,10000,66187.10884904861,0.7634526466836735,4.757924916792889,0.60174,4.3246,50000 +4228.070791721344,12.361259937286375,63957.24264502525,84895,0,63957.24264502525,0.5449,2.5498849609375,10000,68333.47369885445,0.7957589285714286,3.9244023537149233,0.60644,4.156065625,50000 +4333.659548521042,12.685445070266724,65951.39333939552,87519,0,65951.39333939552,0.5401,2.64440703125,10000,70437.01794242859,0.7683952487244898,4.895434009785554,0.60802,4.351689375,50000 +4454.272779941559,13.008050680160522,67952.03266215324,90073,0,67952.03266215324,0.5335,2.604086328125,10000,72562.04839730263,0.7809709821428571,4.373063690808355,0.61414,3.89102,50000 +4579.605948209763,13.406023025512695,69945.19483971596,92626,0,69945.19483971596,0.538,2.6560337890625,10000,74684.95734524727,0.7737364477040817,4.867963518415179,0.59746,4.616288125,50000 +4704.600276708603,13.790885210037231,71939.20494151115,95245,0,71939.20494151115,0.546,2.6514984375,10000,76808.11458563805,0.7448182397959183,5.682906793088329,0.61956,3.79448125,50000 +4828.478740930557,14.217012643814087,73931.97438669205,97839,0,73931.97438669205,0.5366,2.770053125,10000,78929.10322093964,0.7731784119897959,4.878094731544961,0.61604,4.014446875,50000 diff --git a/logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/trial_1/events.out.tfevents.1789267244.59c181d2b77d.38.0.v2 b/logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/trial_1/events.out.tfevents.1789267244.59c181d2b77d.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..ae00a882aa3382ecee6707a5162362e1ca438740 GIT binary patch literal 104125 zcmbT9d0b6h8-~kR$CNR1q0ADMc}Rwk2uV^%Wo*)fOi>6?85&5EQV~)~GDT*YsfaR` zF_}r_`#Q(F&${`{nS9>}`6eRyVysKZR{Qv)HfTJCQ1?X~PG{=MtW#V+&R>|7jt zi&u63oA&i^=1Ao>hhyedq;+j4QWMp%cXD+#5Gg4(QfOae+?pr)W9x{NMYUaRtnIB` zTx*!sc-iZ3?QU=HH_90&{?>LBX^52Px>(!JbaZlY_%EQ~&z~U>(Da#uvCiLsx-sfz z%H^G7CI9E0MIA+T=Gr^Wwzi+?>SpKsUqr=!3=w-9KeX-lH)7w?6;qYx8fFJq9KVWU z*KVRNqWW$XW75;w#>U;n+Ggp0Il5Wv2SCo$*B377{>_=8qiU&aI*H5aB+?X#Bst`G zH0;M4k4L``y&C*CW7e2Y@0D$*a~awqEm30+Yy0`O)^78i=mT04`@26J(BzwuQT_jh zZK~b(lX9pH7uH?WRn&k8BabT0h<8*z1G*j=_BTi&?)*XJpY~jkuBf+2l?ZZlcbI9n z#M;5x-j17g1(V^1aAa+^U%b@!Z(!l;YtxjUyK#YCMV&?U-0WQ4s1bkKBLGLdq)>e< zOn&G}E=Ih5#4#jVM1LNdgX-`e1OFaL%MS6D%4NP>gqEngsG&H538mk96hO@z+tjh= z-zW#GHYUo2{#;ahk-Dg+YsCrl-|`t^^UeUu(+?!yhGp0!m@AhoLnpWYFJ;^RFT#rH z|1T#+Cv-UE>`d4$8u|Ay-WAMCP!8P9<@6Kv5w)sV$PVuI)^2vTGhN+hJIr@={eRv} zxA_jj2iN-Lk*J}6LnqpKSS$C8=0bamv_(zH(EtB?dx-N^8idEVXIZD;27lw0esMgY zJp2$Brzg@8H7Dcjo#xJ+?>H9{D{6tdqxD`zy}yZB7p)4EpCogM!kxftD1=m7{dmHF zzafowynU{`_%s*NL)1;A;_PB)=ipp1{4;Hw+#TKin*{1`#y991t@StQ;`ON|%7z7o zA%A-c)fHFo<}J$jrJXZiUGFI3s^#p9ZPI+k!Fc@o#zn|I9=p=2?Es2ub`}zfYIJXZ z6vLyKZEIQ#6vYl)HJjo!zq>DOpwtg2EIqggVG0G@V@g-H#JQ-^#$G~Eou;_AlSh$o zdW3Q&`utvjz&078fB*p^p?Vk8L>$bMaAOjfA2GO>uWUk0Mv|<`$p` z6mivT3Tmgl^=$h{ph#)NMF>-PA;o}$IU|6=;Mgt+MNOK*&W%SgDEQ$5pg4b)t7cOa z^5>%CFFQM+Xn2K-5T=MgiW^5pwg3wI+eai6wP=b9^LZ4%+~1f2MMe%+&8DEvi{NgP zI|Idwr(A?EMLJSgJ1oOtn7avaJuMFii^JObvr1I5@+Ts515T2U%ddr)O3*SsT0)_b%u9{6jo!xWO6V3s}o4Z_uFoil&Jo#j42Nd4p z+DIts&=ih&Jc@gt<4k~}?-Z_@O+n4Y^21KUfMVwiE<%_>8!6_lP+JZZJwkg)DC*J_ z(ffE5@sU|gfTGVqu9{8pir;Cst9{7=iu$Lx2w@6+q}ZBrrvxaj4je6^s7F&Y;Ty<| z&!QPXVKJ7gW>Zjmckd}B&wygvWG+IO!UQS&OSD{oLT9~|grYu8aWbDb7w4N9T?dL= zQCu~ff;um99(30Milqr$gfPV{q_8Z1)(|KP|DY&Y$KYgejbn!retV5-5U>Y?e@{&=iv&@+i9LS+xg>{i$3vn}RyKZC;Jj0*d$N zxCmhiFQl+pwsZqfJXML3P&A||hFS3_^!=tA0)>k%SIwrNcD>fidf?gpaVi%fOyP?Z z8$zw;0>vSR(-MkCG==$c9>wp#H>ZGN)KadRO+lUAL*iN+z+Cuwa}mN60Z376_M53d zaUt}sgrYHn!j(snw|e(_pt!e>t7cPBbMYtse0!jnb%=`)rU*fbg_{Om0}8Fyg%XM; zG)0fSJc`8zS5^bX22HM-O+odjIjLjn0)>SZ7a>d$ffQ@)ch>-l=<#I|il#J0v!gr; z2hX;*fg;b0tCo|Z>}i)LKrwbE7a>d$ixdMoi3S5jz~1^h#U9m+ricpVQM6h9DF-O- z-{-2?6jYDu@Y&N3C|=cR@ZSLlQzRfoyY&V5cX#Bmj*<{nnqtEa9!1SOx068Od5NoL zQ&8u{&E?fMiWIK;_r}9q9P48sp=eH1^qk70n0zG04=4(Ta@A}KYA(F$ zny3QB_0e2}Fhx32Ox&P*9Vpa#&XiEJpef?{*FdaR9$E?%5&gJoHU)Kdw;ypc9Vj*r z<06D9t{}zAM$2)(ST@g1LeY|@Nbus#Mg7F`6riwmDI&?5vW3|?~)!W2)D!f5QwWk6v*?VyCB z4Nc**fk)BgP`xuiQOAa>W>ZjqB`pon&IF2q4qSvV#cQN^vey48P&}%3Swhj4rieJi zqp;B)hJPjXug6u(N%6JIH3gs;p~^)FQxqdbWWablyG;~dN+{H5ilS5=#U|bRiZB=b z>vGj>3hKOQKKl7dpjh3Wix8&xi4=vg-_u|&M0!pCu3Aot zy_agw1`7LRE<%_>0dGbHHmh3(6p#MY+a)%Tb~ME($OQ`A6;Otq}VK+&LnJIQghrztiC^5$Y_qw%wVqGvO%T26|N+KOsGai=X8 zAxxo!6mKpy-U1Yd&J2)Hbf77&M)N3+eDTTxiXJxy{#VVWpmx0@u6^-3?YD%{$MLP4h@2yd>BwPe9BeJNnx8&M*}GKz2+i>DViWf#(KR*K+(ErtAwH> zP4THSkD|e8*K(j3^qs3_Q&4C3r=tBIfWot!ix8%0juawavjsp=tBU*OjKwp1@mC(?6Zk28zJ@Ts515T2bMv z3RVM!W+4|LOwk!B`W}mE4-|>J)Fc$$Xo{!fc@*)l1~mqX_R(B5n}RyKCtBQe0g4re zxCmj2?nn`BbQNu|yr{2)qB~8oYA%nW&N0(MpztW+s@W9OdEsqy^AS+It9#PUjr%bacvh+Xj*X9YzpeUsOI8^_gJsb<06D9`XR+$s}qSpp*C)# zgrX-+(TIONDtZ6Ui$D=<%2mrrVVIZp9_FHs6&E2)F$gKFPK1>M#eCNY357OIaj7rw zy!gCR!vZL#ui&cXr1<4*8xIuO8@UK!3VoyqN&AgX`*z1A2}LiOqPISeV)OF#cqQl7 z9DI$zoiC`{%w)iHY=uJ~J=Rf;;Xw9_yFc(Q0Ts515 zx_YqJf6f@7Sf#~92vdwk3hjkYRs)4$W}V&QT~CLmDDA+Ti!ZZA4+M%g54dVJ1vM94 z|ES~NdhheO2w{qGNb%%h-Z!8a{Y*_lp-WR-(c@8kw)PwW6n)-u)oco?N10jaU4*&t z{lY~EQC2lqDtih<=^gfN9E zQcOyB%>#-NF_R?}eQ1hHPaoqGJ&$A5Vu5030#_|3#f!=1Z-Bz@1Q#JpVU85Xyhc0$ zip0iV5{kYwMLR{_ih69^_yJH%9>P_#DX8;OLW;K0vt|QDZ^bwXMSq$ig@2D# z`-014p!lW2RkJCm^P=A1)_CmnnLRp@4R^6cDEi- zyk5vv%SkcaalI?d#Z`AMLYQJ6QhYC1`3xvjv))K32GSJ8t#~V{b%&1DK#}*7t7cPB zoi;V#X)B-z{LDoNQ`jR#(u+$LK=H<|R=9ZA8$?s2ZRAmmH!)2E3e!bgwVV_q-gVy% z6unk(5yBMCNRgdYgiqWJ`q@fCF_@;f%cr<+y>>NF46fVyziK%tzAC=o0~D$4xd>qj zH>6l6iq?R+urTZ?IgTMT#pnUNxtRR$Ko_7;n!;7HDX6o%Q8$-^K=Eca7a>fs1S$64 ztDOWCNw+3SD2CD$MK(N&rAkIefZ|&YS1l*S^mXOSfa2UUE<%`MIZ~vKw!H-uPP2U^ z6#6v9rHMR>Y?Fj`Krt$WtCo`@)?fzy-L3nGix8&pLW-CNEx!Q8#&&xo6pXjhj&9~r z^cIQk0YzpPu3Aotwg$~`McwMfMF>-@L5k|H(!gU93IlpB*68vm@(xV10t)>| zu9{6jb=qsKG;YI+GDzYggeiQGVxg+-OrY3%^0|a!I8D)@Esx?($_;$>!1^Lr&8DC{ zYNcjvah1WoZ)#G|NdF4BRyDD&Z}*%Z`V_`4=v zg}Km+FFIwtLC`Qo~Y6iTyP>8j`=LXje;Huda)PC{nN_jBM#S{}RLYTr2 zDGJ`3g#yLEezPSMhBSrKYu;Quhz!n!6;<1St7cPBR}V5b9r6bXA0sY8m|_c3^i|5l zXPfU#-729NO;g-$%$tiLsY~#A+Beo*HJgH3Q9)S~kz&C@E<%_h04W-8^~#62s9AbY zLNSJ>=+liy(J4psJQ&63@#Nf;)ptxS2ix8#=LW=AMPK7`bp7c|49OGz;DE{8-bAVnRUOL%QiZExcU@vdh~Q+#~Gn~S05ozj7#FpjHcQ&2ss z_tDMx>_O{9E<%`MCsKqQ&}{{Cv0#4}3B`Dt;`uoqMQY2Pmw+NMjjLu;Q2WJ3mCg1r z7rkzC5yBK#RMTs515>a=Pprk+3%nao89 zQ-mYM@iR7fC1*_dA_;{FO<_>Vn~NK{lksibs5q`#PKqT5`fLG;hsU@GVTuT(xb-+7 z1?Ix=(pm|{M4Do95sxC}#`O;{7Z+}F)ohBF{I{$6&!49U6ryY{LYN{FDXQx&SPB%~ za)TullW2-sM|c#2LaO22$@wq1YBmLRc8^a;j{=IKcU*)pMHEtaUm1;W<3^l6BB7W} zQ}pILZF>96?=Tk+u5;CF3TnUDJa-%3ONzYDMF>;uM~V-VtS-S^_^I5LP?*vb7AJXg zVYAi~ujCwU!Bw*(@yQ~~)?XzQQ)mjUC?18&pnKUc z7o$3J)oco?(~ci}5wDqh=x`Cj6bFzZq}ybCf?{oLrM==^&y1$X3FT45Hr*Ky6ek;S z)ocoCMJeXIz<0!QRk;XZig={xIVHLPR+MFY3kk(kn&S049>u9`4e@V1)#F?>n}V8) zZ&!BU7XzX$auLE5hmqpIAOm#TSOqN!g*i=eeg}`@#MYkCFc-D!aMf~B9Nc5%2Pi4uxwG==3a9>ue|(}n=WroLQ+FhwF# z1n0SE0)?5OhlIj{rf`1DqZsre+Xg7!PvENA6qJG73wz!HD9)L45yBM7NU?802mE5d z=vKZGis>}PrF0&JztJ=^pjg|1t7cPBoz_-o99|pz(2a`_rbt1GR!1A-_duGJ?2=Ge z(iD_|e7$N`7bwqc+P;;SjCLHf~kBH(TgelUH;@d-w#&BM24Zb6xm_bvVir`V~>2?|^ z&hO=_<)pB3oDc|e;hfAx2veLwijxukUxDI>S+Rs-CQXrgiASNM8Sxb6;?*p!noU9N z7cRf1YygVl`CNoB#Tlfq?^C`FC_Jy!j1=$PvuKJ%pLrA=eP+4>#qm2_wVV_&1Kp#6 zBKr{+Axx2u6oX7g3|<6m=(@JPZ_H z_f3;f*wPdse5Vb1GB6w{{v772*%Z`%p*Gsm2q+GxauLE5SCAsaODzh%v6q&ca8G=8 ze`S2k>rT{7e6%QclXen(qVaiww4BZHj{lc;_nE8r_WAp1w+U1IJ}Te;D9sQCDZww` zm2F#Z03cB&*E{-De01_PJ}fc(nRa~6XN^3*t59i%Ac#T< zf;{VYuLFR5Pr1`$ZN<{M!1y4=nRy`yB3inn8GyV`mzK*3vTAjEPXKYhD$NiC(QiEx zme$AdJ<dU$S(rTVGLlDGkAcAN$ zN_z$$VjW|fp8^e;b(i z>%VF~(vZqNGP};DJs#4|@fi{D&CCe_(ll9Lnbj)UjmI_9d(RN~D5zS5H0hr{Gy6t= zQz3j(RpF2{QTRKvn}dr6!$Mp;`F;&pNt&6`9Pvt`J~aEg(@vyw{BdFIpueMj<@mly z%CGaJ8CCq!tm>`euiAgJc6_@%Q+eS>X%_vd*_-F9<9gibHQNT(<3NQKy!FWZ)@cI}xp{QC}J)@5g4T!_$AP`bz(AYG>82JCtMFN>eNU+N|G(ow(O7 zxo!1q_}{#vYc(U3ANQ2z$@jV027d;@iYQ-^+aLPE9ert}U|+aB28S>4XP15O@nM$< z(v+%waQ1lk^fKs``F-Ezi(5=Q@^HA>&~+8NR|MlT0C(msL6AM~?q3HGV`FK#oFEoWd%FQh@KkAr zAc%q|g3KFb)CWL%ANajN46>c^-u9Fc)(GN}{NpWv95^N|XM_Ca`-DaLGj9O7bViyX z2om9kAp76hD*}kawL#U^SImen<3-!NO*0TgRVf*LV&;8mIUA&$58_$=+z3GOA4@X? zLG;6TAa_T-1Q6G5HwTJAN*V8WF5WN|L5lY%+=Us*)svR9L8uv-zo*ep07)Mr%@726 zdJsWE`*gbk{rg@+=X5d1Gc8#@F=p#t+`QMc8=?e0Vctd>B;XTn>cKcJ7y56?0H4^Z zAq}a_C-kHC;2~A35oZNH@wl5bO_om#@8FBamA+N&D)_`EU1`!k`NT-~8biP*ehic* z3j4%Lzop3dK_Pr4tfa=nr8(l2MES(qfR#u$#VAM-d}8ESX+{-&!uHxrHSmf2iP9|E zCsfCJ;d;!>YY-0WQE8gAUdFfUSMS?`LD!nkItM<{dX_XuUZ0>2@4|NTUEmWl=1Nm5 z_lY-G^3f;O-M#hze8R&?nkSb}%o=hLSA=Qm+bHM@`xZ+h1^a^1NgTfC?RvX`Pn55e zrc})*Iv#wD+pzDcedok2rkL^0uW82-xH^xG$YOB|eMs&-(mH{ho_KG8wNyJ-w%mAo}qK?*PaKOKCY9gxZGPO>^%6h^w76 zLl9)vTLcO5agBo+Y2ntW?S_h_mCkr?C3>j_f{YJwUj!gNE2ZUZ5Nbvag!LE?Ankpn z8G;}RB?zMWI7tIQZaQ9GCk9Doyl8M{bSnfY%;|KT2O=$JgHSVa{>C^<014Y5%@72+ zT?2oD#ciMAE7*v3J#U^P26;4GmKfd1$;N)%Bv&UN#K`S~G)O>is(-11AGgBV%1lV;Ik zbUf`0o>)Q7P7`50mUNcZ%Mhamp?5H-_mj|1AV$7Dr9twF5p{S?W@s!2F*@Bxnp(LS zX*Ow$SDyEEul5ea=-?1(o?K#-@N5MBB+qiAb$BH?ag;PturJK3jeVgrFL=Cu|~gkGs>KBdOSil!gSc=^Q9z8S904&E6LpjYm&k(RT2CDrs(PYt$(UOCM{ znjttQ5p{7)7QJ!a2_W6_>@CC~VT`v08sxS>kah3wmIBC+LTS02Ahn7!(I*ZUOEUyP zoSPs>se#!70I9pxI$I19z<9iS^D-p_c|W;{Ie;ABB`s%zP==?zal#0gk)!*h8G<1E z9}o+@sDP?!?ylQtQ^nHqVLXqU5>pRBicGhd0EmW{w44n>ZNnDlVm|PcL^y1Gv0O@#9TFwUf#dmJo8JltdWWr5ph9Jn( zZU_=!w(lkM@2&x&H)4>R=CXWZ#p@c_#(JJwJq>&!aHlj#z$ca%x#PHOo?CAc_{8hI z(vZr0qVC(@xYr)_^6m;g(Kb$+Cd((x4o}D9>i^nNAARDeH0hswqQR?%{lF(Sr%Drr zeWKg?EMyGdm3k3YQtnx4j(80ES08I z%_l5X8y|pPsq^&EPjQQR&v-Og!}vS$?Az$o9#;6y57KgWucUUbSF2Cv zNrDcJiJgBj{x!V&jGC!m#qRZ#@hs;ln_mbL`T3qJfV8_NEoX!LiP6#J9g(ilx7wv3MmJkaGpZ;?)dIW4fEbR>zF;w08Y$Qpa(I2AxMY7V5Ti2_ zq$yPsBfZsu_;cDV`;48irDFGrT15BCLrHC{ARB_D?Orzweeptz z_j+lDAc(dLg1j>gMO8KJQ@TK-^rhu&5K2{l>|8b#W+Y&Y zG(!+X!4pB!c0}!lwi&g^^064?tCu28a;EG7lI$!j34`6&YXwoyM)|>CT*c?(H2Nw? z{waKYZJP4Sbwr3D%WE%WnXYVl1GZfYuP5~bDpr@z8bw)7A!^$KsUGc8Md=*sMC5a zh^T+^iYAT2jbQ&=aGHn|7K;gOzoS@0bpM0D3#oaP2okR-$}HSJsN&EYA1K=}`0t9E zHRjWM<&yhENEOv0GilDPUVp=Odwai8Uip*=qupZb^+C8W2fX^F0t<8NJ0d`aTmKh+y13Q71-=F(pmdCyfmj3Sf$dWNw+mFGr(J%*D9(XT011w?oL_M2@+Mt)$YZ?wL8{LTzL6!(V zWI5iZ&qQGPJWG3}nB_tMgC)KHBV|dm?d>PgGFQbb!54)^4CCMS=yGUmy@N*oV~MvrKdI#BFN&r9a(~!YV zOjwxLe-HsOY$J7r113#bQu`vj zKg)lM#4$vO;JBEF;J944 zGp`n~XxQ|e(#ReG)Afi$lP&BZmkPWa%7%ERp8^ z-+-m{+e{ZRi*FQzCGN;+WVz~b%>d@**B7Fm%|cyla^K#>7+93ch!8;*l_+F6;iYi^ zSXA1rxGiQWjbX4{^SO*HLsJ@00v7G=L_M2@y597r|CS73X{bkp2(tW)K^Ds&O535S zXKi}aXM4pOyL(WkZFF7I6sdBpN8z(yYlDa+0oxF%`SIH0_w&oZHadh8DV5oV(@7OP zsOb2CH^4U9M-y?fY$Il&F&gGwsCh*Fxtl45p^1X zZI~AjfpXc#bX9d+v{t>0A3>K;FCsDpyF}DM9KtvLn&Ds@e&2|gs@cZH$fLcWd#)Z{ z+E3hZivQnn+KJReHSC>ST@6G^ij5T7*Qk5`BA(I#tqfPfT7Pdu)U&%Mb<0GLCo^<2!rLS>N#Xl{jjeOu*5Ya>e(#RuC_TKHV0Tt zTM{9HELV=;9G}uXc>-806{dX^v*aYPSaMGx%icSty@4fCMAXa4@?fH73b6P#CPD;R z5|WT*$b`hYz#`Jlw%k#%t7RT%u!vSABg-n^kIR9@asW}!W}%E$JJ0hKuxuDkgb1>P z97h%<|9dNdW!S3osbZG&lMI&cWhap3t8I^8z;bsZQO{CsA|WAJebScXt7Ex)((x$!kv30Y<$|G5~WreUON)+?>i) z=J3|I2nPBHQm6RYGx2IRWqk; zV~l!0_w-gR4ia~q*bGMZ-0Gf=r*!z;9r)Y!hV6)YcK4)qf>mGhf?!-8cOgOq$HgQA z$K}X_7z<#z)~eoPF-zEG2Fs(tCy^zkt|opHJhLNF&t{=^g6`)Gi-F~rHW4Do((f{| zc)uTZ2UzAb))^UGv8x4KXRv(ulY}g(rGA^C2W@Oa)U#QrdC|CftuL?~>_~(NvZ!B2 zmYleNcEFOcXY4*Pi_a|vOJhDuo=J3VU|AGT)U#Q>@}JvCRX0lpmS-tMh#-s7Eo50f z1jXhqnPFAT?R{{?QxtJ^$vQfz|v$1QO{LMql+#sitQQLw!uQe9ngLl5o>{Wwe{3E0N84`$dN2XDB&5^Q5h3XxKo zZCuM}gaWbobc54Mz8%=crawfWT(+_D(FB~+&2orJR3L%|Asb_Ou68rdaRJQnt!yGhkR|;Q zvg~i(P!U-APJW{kTCo${&ttHpFXBB;ov43u6tKit67_5r>e|LNkIPGd#mtTf5oC$Y zLl&b0A%(#5`E94IVwNk<87zeZ_Tap9x3ND2EM;GbdO2Cf{#e(!m(ca8n-C#_EK3TIRc zW6;!xU%KwPvto?}zLjYkDmS_ymC>R;WnddQ4~QfI+t7@&$M)Ez(ZFh88(~j~l*(*l zq-}FNsDODE9li*~XLuyw6D`ZcZHvwo&T~5%o{DF*>#XX0VMBWkjT~ZJa2r zjhKVq|6UJg$CcWd(peO*D9Sb#4CB4K|6}KlYG4}+8WAB?w2d^Qd+J~tS6dTdv~Aq7 zh{uKLHL_0%EX>~>hyWS35jy+?Ce^h3S^~D=+Ji`v*EXmFG~Kgs2iQhbUm~`0+i-k# z8Er%1+^#-g8`BMlK)GzATHOVBI@u(vrbCyQVN7HSc8T_HaR`5%xc3Qcqs>$zrfRm) zbJg+B&^u$%x@4{NwJwep7yC?OxTJ{FTMKCVoGKdht zahb&%m;EALYhbbWXqh2qi7H~SgpH0xmf828oCTK7tBHCx3uPPYru)|emfVd*h#<@8 zBAnwMbx$k+mKHAkbaz$k1R~f->&GMOVPJN=a=KzcAS|UV{ zMf)?d%!pI41eQa3M%%CskmYq5 zvY4F<&Vdc;TkiF>Vix~$ndVer+8JjkAiQP~m{XfKM3R6x_3O?1e757XiibLUK|8-`ZI_Kqs>YC z6YtjR;;s*dz`|^7NCe0*r^tI(af#<%JMjw4Y4~I!NnUfJ4zP4ru0NPlxD^pwxj8lY zm4)UsWyeb;Fek-@M4(*elzZ447j11Xbyw&TdG17}V3%lHjzg%Wop=h&>C+k_rfTMt zrs{qQx@VIiHksm%^LlkP+MIgN*oSLy5+Q=) z@>;q#lVvGWBXe%%Z1Gh7K1*#c`-G4(G^(UDejQgvsoyk-O}ZL0I)n15g~#s z7iuAk!|hYuf#u;bFNg4oHI^F4U}+n_4d*2z|L!ednUYS_vstL~<;TLq_{^5(RU$-? zB~=Mo?Cy#bps8;kw=G!A60o(JtT&Tu_S8YDZe^G7yFPhlM3R7QoIAz4_fmiGoHJk> zdNYWW%50{#1$OHlrug)3z5@~UPquM=Q^ISojUnzt zq_A!5+bY7bAM-)G2v*e0-o|cjx7I_XOKW3L%o@wGHY3j~&g70^8WVhls7* zHWu_MK-=gTd29vP#-d>n|jF)C*;-ZAKNgvb=^5-Q5rC3fw8w-jt+c?uCz zHQP}BV}_r@8FqZ&198WR-@)jf7t?m*TDN*JuR5&tp67{rcK4+IR_op+81JdKxlV)# zj!V1<$K_$#_d&1|EUDLF^qz{HAR?5(a&}k#E73xHY zAWKAD9*galzQE%1*(^rPvVAv$nOd z2(tL9Aj<>Q`U8RGA5<8pU9F`Y0IZKFa*#)(;u?w4sBmp%I;)#kxd@pmWo?wsl|3fNe~^ zMTF6|@wQ1W{yDAry88lHm}Xf-fDGIC^@Vro-ulL+fnXaRPlzOWZG$?%CU)Z+fNk7( zNyJue8xaZL&^G)^GW5YVlHL=6a@mH#tZ1Cm5Bb&cj=`L-M5bVu=+zjz#I)TCE5SC7 z{U&0nW*d_}8lr7D2hYqBcbwu_M)%bC&ikgWQ~L+_cjvN5qMqG7i}^c2o|DC1=$=Cl z5+Q=);@kwsrSrX{FR&Ago@g^_Z^cgVGM>TWRKVK_0(5HQwT%J`qMpq{nNwcdiz|R7 z$A$oJ7uTp{Zx=S$kT{vgf2s+wdKwi>qzSgSRW;PD*SXktARnGc{*pdpt61@=mY~ z=c7bQWwtTKi1)g5(lOo-e|$PY#L2Ra1$l0GWOj2bHNiGUoF}6G$u`DDd+!C?Fuh7d z3fo5V6`pOlr8RpFXNTKuB1pWVDBCbO*BAk}b!lG0o!!~vW{zh-9_q(?x*hY`{M3TI=K^>r0aoQlTjTv8v*vf5V z@26_`asS^p?0yr0a@oeb$wzTcYsP-YuO%5)Unf1i1-rzaHrOR@9qD!%Y{RJ@ z5mPnWuw0+_6uM{Q&_=h#9Vht|qkGo#ZiJVQ2bNnu#*Elku@l6mGg!3awqnm5sH?jWSPB)#O6%Dylx_TUt&MM4 zPpU(N2(olmN0ztulvRP{;G02lVwSKA43_;xylWfXb8qCmr^1` zkfn8dWGO8ikJkgntag!4hf1o0qU*+W3}rvOiHTC(DR_lUf7I$S@*AkVQoU zSrp@wd&0a7SiIXUs$z}#WHMN~mGCUk!+O^an3urSL_M2@x|5&OvGU>m!`h$I2q7^l4u+vC34E*HQy6k~{# z%53A?-d1=z=NHXOglijJ4ij;*Y-83G-aVdvic9hRn(`DP>Yr@G)8a${*hbs)M5M57 zoY7WBOp^~gLt#ZFXA(i;6-C*`xTd_fdB^8y4FKEdoK1vO(KbqVDD4N^n3zX|(YA5D zAMfF)IYS0r2HRNriU^Ql8}7|{H$L0eY<3oGB%PjA64k>3fs#PZ-X_@%U z2(mowhAc1JDSm}{x%$TbqnPD>E`#On2p?og{M`!InCe%ep3Oq-(52V!wuE`fDkVY$ zS?>2h7Kb_K&4Fb^w374wiZym6pTSaK&SPnHMzk2_<-!r7p3Oq7u~WwW&4J}|DiI>c za-}D-*g7rV1T2dRx*ZX-oO;1vIpDb#=Vfk!qY|*R`b5;TS*SJEclNIBz%sgo2oYpC z)eBjQBCcMA4a(-h@0Vhh%P(b`Q_sRaNM%seNgvE<ohSnOQj?APv(@iATS!tX|W0s zDQr$QMZ9NFW;b7&04pk2l?W2AD9W642Jp=3?*Cum0CO6zNrY6s!t@zYfjVwP8?iz3e4%#SR%G^ zbGq`r5t>u^tD5V;oUTqG0_8HN&K<7dqV3dw^L6MF-K~jC!7gFg8@t5zhDpsdUlP&d@!z%2Rs9RO|!^g^cc5|2FT6y?a4wJy`3SO2?)3?CwcDS*aFtD-^~h zO@#;%9G3(g9G6I^D|kJi(Km}AF-z1t2Fsju-rvy2hacGoEWK+?m)5gcsOJN;AFa6w zEGHWfA%ZMXddTA6dO$yzmltIZUx-;kJ~CMRTKi-FPxKgm9#~c=J(AY5St#2WY4ixc zSNp3W5hBPE(id5>dmJhPmWBtMY-20dnBNx$OUH}8$Pzzp{wHA3OCjpnEYun+JQ0R= z-YuO75oGb}k1R=hJn>27Ix8CN6|;E#V6bS#u11#5Zxircp@8*7J)4EPTNi263D?-U z03t+?#cLq4{8*+n7Ml9e37>z7SyFzWZ1_4?~vgV&yKrq4Yr}Nn@Ez^HmC#iAJA+H*hb4J zBDQkd*gCis+QzZ1iv7VhvJMh~a@oefD_JW8%NZ=s=6NH_)QP?E*|EKAh!;z(~W;=agF>NzrlbGegMh44)&#t(}BD$^60~YnJL_M2@GAF|XjR;_g z>PLhKvWyssEKgq7n*g@a{QaZ`2P@_!W=nNh@7FB(#e2Ud`DyePunpfIM3R7QglyP? z?QzIFzZYN|9sdw1mD$F%3A`s>4SHC{fNd1jnk>bWWg9yV^L~@?ut~oxu#LG5h^T+E z4ei>R!@xEInh}x0w$c4Q@7l(e^G6C{MHRLog2XF|vW<`}O>pR=I*d9GwxOpFkz35cW_~D*C7IA*v9V$YPiHdKd*KNY(r@PktDBePzR`% zF?Ax?Mvq}cY~{9Lv$O-+#;bPSdVy^W8$$%jWgF!OU*McxZZdc!bP1h_M5bVu=wygP zIMVo$2H3^~b0Vf{wvn%9@Cmx7e#CwiamPvhf5)Nj3WZ(ZUFF_sk~9<6dfPanp4~mE zE5jr68t;Q~nRS#15geC(ym3+2-rxl+XIC375VOQ@Ww5l>@Wh=UDoY3dhJLY$sAscK zJ3-S;6O)0Z`3@pPkfr+=oa0@ZWj4UFxUt76F-t@cgT*m@1G1RcUWPp=whd7)CyQs# zceQ{es3Q>~$f9n9EX5_wY=NbIL&X8{6}#H@oeY*`S2rTdZQpSGf}BEYqFzoGlZ|Tl zmbIG(5hBRa#28un)@_R4L>@n@riYltH=Mz;<%~13IHq2m4d+Y10-~PHLR~*gQf<2t z=EdBN2oYpant&|ou?H^0y!h<-k}GB@-OFG}UCKN6Pv*>S2rNSn5cO;p>U^njd0#%T z+({%t1X)T=kmYFIQ|e$&JHmT)K2)*BHtmyXPQIgf*J=88wV4U#q!33W37C_|*p=Ao zrysSn0dwk;K%`V=PL|bqcZH0SPkjV)T5_C-lVwih`||Er#s2t#Uy#c`OGN#XIi+@e z{te9O$5kRy*ql}^sEcRE!ok|(VMW!sLj;Le6lG3M?|Dz}CaoBScZHVa5Fu4GCq4DD zX>gs!DxV0W&B@c0_gJ~X8NI`>Ff9v+02$_F?aF)T!MWbp&0tPf-V;gkniF+^x#>?W z!JNFl5V4h;)5xFQ(40c+ov{OR()mpU%4JS38W!W}bhXsT0=h(A_1@CcTd+%PnuuM( zwV+8dn3H-PBBpBQ6x_i$8M&(LMK{+Kg-6-_FSt)_S%!Q7>or+~oQi z-`H+3fCv#Bm*Pn{E(5#=djiYc;nNbtEHC32EZ+OpAj^iIDki|Pz?i6)lckY)$vl|j zj%GxNAj?ZrWcl_sU^=iwWUT3VxMC;BImBT3zK{1j%^bUBLBMkA7E#Y;p`P1#f9SG0 zu%zS=A%ZM9X2@b+*M2y#{I=-5P|T8fl)*Co{5+hOt~XU01Iq_nqMpq{T?^c7)AR(e zoOd8X1X(i8k;VF1=SRSz*}?X{m?ixfgC%i0kEN{ZNBlL3Z#{^5HVb7=B^FM8!1A>Z z5hBQvZhP%|e!#XzL_kF`BOACT0n^$Y2@rkT)-Hf9I+Ji`g8ap3OpC4{)6GD+yS# zoQM!XmVnvF(rLxlpTIKu@y|Rli_cXCi%Msn1y203_yDlX5-r@h zu?x(xrUnrr$YMVaS$2Nva}rphn;MQttk?;j-e<5}O5k0m8L~7G?+V4J5%p{qYKOk3 ztL_0Tn>!OBf-JKZAj`4w#ovJCg-QR_VwU?k43>2_SK+)E&d|fBs(V-x^=uYuja7T) zhu;p#up>eQSxoJb#r@ogEMQrEd+u8?%az9rmX{%mk)@u)xOK3`9CL|!HVb9vvtsVN zhIw&+NrVWpjCMqpt)^M{#OwWgD~*#X*4X_#nYOW2V+2xts8=fhY{MmwND{D(>H~S- zHfy)!_6D#Gl>#EAGTR7t<$dEdYVOsAU>o)yh&WlcQL<(YZpK%#XTAs9(Ed(D{gZ91 zYc&&p`A@lwh!nOB>Km^YUd_VqDHzpQC7ng_ilS^IK9_fGqrfw(J=lh7T_U84wh=e5 zT`<^&Zet>hwvEdvyjK@ueq7f8+wg2j1jw+BT}t}6#3zi>3j^EuqD~~qYa7%7UK^bD z5^Q5|XCk(8+wgK3g0?ZJOLz*{#;2Y{pj@_b?Pp!Ijolp@;xGSY^(8U|yM(C|4&lmg z-)e(x{24;TRLwTF?bT}w-Sc6R(jsxkIrWs$J=5z?!9LUdV!h4KJ@wBJ_3Z9R?P_}a z_TXEi@mGit!EqVnjN`K2@o5uadH%}zs+c9=C4{^v5-SwgAiGF+@F^g)*ni z(vxT#D<=^lf-D+tJeFRah5(DPR)_UsmXP-hmQ&ki;=H(c6+{8cyM9DHn}yn;50AKA z3iI;afCv#}Y3_k6r&8La0!zE;4L^%n{E8VYSqH6<<%{ufe5(4e9Z@eQ%lPGW@b}8+ zI}#y+EOnP4%bXO&Nzl}jI!u^#tYVFw|18rs?wA`ORchULI$#^3oZ8(2$#XWua z;;VDPHvIb#DV5pAPhZ|I{}uoF&;)Gb>R=*HmTmMd=-Arn}bq6I{N4BIH4JQ|mHsP_^4<-bB_B1vA`pbl`0N1N+l z8$*^5v6b6~^@LGq8$)um0>Cz^c@cqf*@k7uCTJTCR|n!R+-0vPG6lPY!BQN;eaq(T z0^3mVCt|8*8(kK^8wuUh%CDP;xZ`+zWpvMxJ$dg7B=2pFU-w!PLe#UnC-rw{;L0Pa zuoJA0B0>bmMZputA0_J9ER+Styz7k5 z)7(lULIhb}uRxYlOKUm8yr_;b+b(8#{D;9ZNXHE4W!SLsc+IJWDN)a6p=_hM`}F6q z#_X+#5J8s5tB|FY=gM$k30T;#Ow4lIy9WJ)_;PjLTh_&6JA8$CIp{&uvstKlxe&Wd z1z47^B0>aNUV9;nzkaQU(A4!$7rPvyFIoS|b&ZruXgh`&rE3E0Nx zy-RV+Q;go10k&aqi%6-=HZFJJy&ciYj?e1c_G^WgAnQwn4x^50#Bz8wb7;Ayu@EwBr`a zaBZV+DG^57#+b3Zm)QCRwP*+nb6xdM{DtY{W^F&y*4@S0ZN8JEfk;WQkwW_#R^hw{ zp>Ir{77W)mYKw>@d2NF_K;Ks9wt{VxsSvT1+lF`T@n{i;uqzowj(kHyF~bE>=N-RPWQkzj&~+vs%9I;3tQlK9X4p5S|#o{7uGYn=XlXn z>@&mnx4|c(=ky}#+1-=c35JxJ#6b6)HIxVu9GB<3ajBNo51-pmc^C9p%#yl^!7}8F z9kR^w_r|9Y%f1rzY!>PbSd-*$vB2VAMuZ5mJYIuyyyi(F{xk|ST6DKB^q|_x?WOf>7Rmx$HU`%OmYjw}h#*UrH?mA_w+Ekf=p8U{kC-K58-vBj zY%;POQ&=B>EW3z$HVbvl>4`{T4X}ho5g~#sw|$Vs>5tDiu#K3jPivj1n3wIr43;w; zc<0OLfW@7G<;^{!p3OqdOIuCt`oPld2@xX5a&0ZLoEDWOKvQ4%Y0g?P%jummZDY3q z?@o$V(!xxzji@prNx(K%?DfPgPqF4sXRr;UnscScQ<-h-{lc@2DUQGK8?Vdj5^=I@ zqid!I9@+gRxAed^&Nm^V{>e7-Lso1A+bC>BL<-x+&QRV{qN$qaR>F#U+nxv#uPDkk z)JLf!U~JqA{C3Flu0%)`ZNo9b;sn@6KrbSUwhh}RychCPf7a;*3p1e~5g_B*hS_W0 zL*1X3I-CL92p>u$$!iei9oq*W1U|+ zv<=OzW9^|!#F`PALS4cud>wX)_C}6R!8Sfy5iwP>jZ)nVd~U;fVZTk{j^n$F(LKXU zc~6PDHrj$_J=Be;XLnEPsW`^a&P0dU@a4k7BKu(^+Qf#uw$zI9Gk%uCJz z28-imD8jsKYeUqtS*SI(d}a|o@tV_x2oYr2zX@4%di`;P4XSgGkHKOV|06QZ z>0O2q&QPb|)x*J@uJt971k7pS_NlmiYqZlT1alfSlt`(}oYqZgjR#fS^YCxDPUAa@ zh?8YbqgwN>)3^sNCk z6lG3v`E7CN``3DJ4cBQt*$^RBG^eb)0S;hJqZblkv^fRj^Int;j_SDs5!PB@PPx8Bpj_s3 z=}$*Ar!h;c&qBvb4Inawx`fwdUYA%s;_~FtrEj@G}SQa$zXM3t* zS4&P|u&lFQfGh@{oy&n`YX_oUPL@rkcKAGvZ+9X@kY(p)WI1koWeKpj&Z`wIW{FK> zuqaQmMV6*(l5>H@)|IGdvrs#9{NhYqn3visi4Z}S?OTv#MPgtJVEMgxN!`;G^AdK3 z!LqL<&uH&$Zi2u6d}}pP&t{>FR=JmPBVgIKkq8lF+2W5ZCLdp@0E^M{sI3ptJpKRmA;@)4tHttR#B86>3ZwBuflohLv;CC-=Y=|K7 zilS^IvPL@``i(~u1Hd+N9f*)B+D6K+rjcM9cRYzO+BVc;d9RtrE^yxp3-hH95g@}h zHhay-C4Q`to&wm0Lm-hPuWe8Vc-noq7TAW_E+V#a+Yl|XL)$Rzdou%ULo1F5l*=~W zR?|Y;cw9CN-&!4#Ok@gniLbma(MKV0E!c+sIU=TNw$amll?!xFoqfM@#U01z5~F)I z-p2b|EqrjzMChLH4iWY2?n&)xU%y>?4?9753K1eWE^7jDX{B`W{|YQytqNwGso2$i zUS+VPtMTp%Y1=s|0L!YmL_M2@GN%J-S-*g#(1{2UWLdrySq9#;Oaqp60UIO4EbnhH zSnkgrh5diThxN;W<$45BFDHvu)3x}EjP3`C5J47?ZOGEm@LDHeaT@!i#@UK_d3uMz zGQfTUvUE<>v;~&FrbIoPg|ZFXZU^tdyiA@!gb1=YZ%38~GkW9#OVP|;3&kw=A23)l zeRxl6)UoP;zx)^NL)5cbs5@K^Zg=rIO}sx5BFM5}2eR}o(shBReyr*Db7Gd<9GSL} z88-%JsAJu?`CuEV?T91++ZgxV5x2aC%R1v-p~hW^l*())x)<-Rka3{eMzD<#Z6Z#V zZ3MjW!XsOf+-N4)#<_k()IZtA)99h;U>ik4iAZ7F2%o{bD>OCayc?Vy8iquWctugR zVPV-G0Z&^d?f~0ZHl7HnqHRnW9QzP#~=zzBIGtS5gY~!j65h#~!9G{_! zwxRKH+<54CHl9SLP?zxP5rjk7_TAGhU>kv}iI}R{M&8uswV`{4&w7+8?l@N-GP>u| z!xq?QTJ4!M2-bS1`9wXtds4P>c9csu7?=1(M2OJ1DA@AGW%T@c_P{dU`nXAY#jbYh z34>++H%nwWTkRlzuXf5pqFzpx`ppMs0m~OxB1Dj7MljCtt`wD4z@ppOX@{64;TeM^ z?({@tQT(E_9_HmiN1~q1Lj4ULc)4H#ux#r?gb1>jhak&>I@(6Sl03lwx0ogB6@%sT zNMmGK89hxKSayye>e(!m1)9}QI|eKZjEN9ImPw(=GR|sSJz%-|;^M4x6>BWy4TB{m zYAmuuIh#hxYq8{7>b8_lVL<*aecRKGn&7&aw zKv+>>Iz*6oMN#H-Hlzar&M(~X4$R4XAQ4hUbLw8F<{dC6X9FUPHmAu$d7m=4_e`?8LdWya-Ckf$olJ-%dCiGBz)Sf)b-|p>rxLN1o74It4>YG)cgDU0 zb80Y?2$aj5Zl?A}b6Qn;>>6|l?YTszP?zwU&+8I4?;F(ua~kMK#8l0kbQ}$Tz)n!j zuw=mbik-mg6Qg^2L+|OIg^7Si&rNuYiyF#1e=A{73_ZdVzn}xa_&_X+MF3fS-d?G}UWx#G^nbmR% z&P$BP@&Yl->z@o3liAj|Lyzrwa5AveUr*G_$+A-W!UteEwT%c7WYG;rmW43^ZD3wB zEiM>mRIIVbr3{wcdAvI*;k}f00*kR7QO{mo7B$u%`)*+%3do^3R2a_VmF0Jaf%i-`Is+o=8GTurcz-C0DWux(J^K?z!T z0ROJI_k;)%uPDkk4t3MOq5sn$`Xbmy-B(0N6>URfajl798&lsCVYF?ut;_o?Ly(8U zOIVnLzYqa3Y-7tv-U}UV-4zCcZRGzVlH|1w>Hu#~yEF=HW2s^r>6ufxZIrxUiMG*L z^tc7s##Ch@P%hiB>^~H3<6+$~_k^NyO?VEr(X%-bQ#IRYHR;11 z*a>!jX{mXkVkfxZ!|0xt>bzgB2>r9<7j(}}&2Ya7*Tw~`)&(6}hH z-;3RI#=w>zfTf1{Nk=hDsxO1Z(P|Rz&}*;9;uqu|+Yt3^7Rok?FV(PwouH=!5hBRa zIucnD2Q+j8mhUNv55z3-ehe0`p}gxf4f>qLcZJ5CBkI{Klm(79@_qs=ZLSd^f-KGU zA3QatZ@%BY_vhOoxmtcZF!StdWai+5y==MW)+EGkjR zl2;m~hAh4|Ys4(uw=-B~De}JQvaf00XJAR*O4PGisPko3e0xurm(X2Ah#-q78d=oM z9SWeSf82by)}@Mh*%K_&HgZ#VGnCgqei7P62$3XU8}qs>!!0jOwayl>jSdk+N@cbY zGnDrXO2M@^!C)K5Vu(0dwy~nz2ahavTet{p<4yt*^-s3pcP)Ab*oH+a5h-jNfh&1W zysk3t+!I#Rl(R&TctugRF|il#+D7f6Ej7V5eqJI%s%RUY)iUpaZTz@FgweJ!ACc(220HeYiUY~y+^ktDBePzQL>XNnrwM%Hs8wsPADRQEyK zcy#@~DcDAncSN9Ewvl>yB-(~gVZagS64yTynL=H{>lUv|G_^DO0=Ch&govq{ZRFaW z`d??~{#R4h$MKL{GAVk{2o1TFn38xxBGQv_8!D1$T$B6DsWGGzArz8GW#~dtx-z;U zMUn~;73DgO$R(kRF3+@o@%`+*_UYsgSfAH^@3qc%z0cZvt^GGt&*&iYZd}Fj^k-Dh zxN6>;mBT_F41}@X<1}f{uAYMbo&7(l`oMjWM3WG)`=X$D8mnjWk%RHT^0L^#o@1#G zWU%NInV@YrkNG+hSQ5)fdpTL^BqeFU()l9^5oKxQvGi$di)XxA+E2K}v3xkeV3~e$ zKC1 z&h)Zc7o-O)2Un2xY!-nzX=~nY11!Tgk`Pgrj}gf7!g)FFuW{1bCE-{K!x=1NrDL&Q z0yDcP0n5}eq&=HO@O#ndF+Cbs7EB}|qAV3>kR?v7G!Obk;eh=ojwL6O!SW@C*Dq5v zr{b6YuK1JoY!-o?zccNOXCPSxlMqpsva`q%7J6kFOi(w>eblp>$Jmp9S!6sF^1i^^ zWo^Gos|%RZ+Sw#Y#GICe2ce%hYE-*|IlZ1oQd%;nk>NaZs@!ex2+T=uF^Q99PMs(2 z!z&9`eXI!Pq_UDkeU~{EKTlr@=G1o`i4-@dMl;@1q3aX+?}J~5yE6&mMwGyus>csS zz`n+JOu?LHY$GA9G$#woXB)trI`1N3LUW40EyaO(&T2aDvR3pa0W!>K`yt*oMZ={j zRbWoxz9dOra}r!&xx*AiFsJAM65FylOm zD&D-aBvY(PNZ<3SMDW6`55b%k$B~%UnUg}%h0{y8UJrrAYSreO9LxCx2FouydFwPM z3TB^!-|g2;q&=HOFss>_KM93C?&?NDL|KZXkVUs~$~a(AXbc!}vw0?nxX55>Oc{YQ z^!wuE!@%O&G4NY^HjAKNW<FWWs~m*EH=3$M3f~f23ZC<@9YV-;j2(MIJhUx(Tl62y%tfo<5H8HBcx zFi*t=Y@^APgtXE&lsoj&Lffz)VM5zbOynJ$G$Lat?o76rNdjcphEfu5%caBi^od{_ zMm8i#UfU2{pl!#rt6&>?_9V7t+ZbW64{al}@thgh#<1liP%hi3)0&00@$*kkflwuU z*O5%IDj}`lRf(5BT}cJo2y`Yft+S0yRZiETdJa(Qxtyyw9vO`4x#SsdZR3Dn1b$)e zQg6~;&g!Xqq_-2?mjG=NB6eRC(qplD?ig2h2v`b-7!`0VHJJ>SL_=Q9T%yw;f#2;J z1Ja(&B6vT*L;Ih6U@03*LPS|onm5K)%c1Z2q$d4Owy;kVAU%W3YH`}qu(HL1gq<@M!>$Dm(^ z6p;387Qvd+(H@bx&@WF)NQfxQ+4IO!_$FgN*haOrk1@xRe@~`u_;=^+Ogr15D}MPe z!GR=+*v5j)K%DX-W}S5e+Zg9aQd+W&0ad)y@w0;qLclg6){{6{wh@x(i&u8pF%-{1 znY5WieV1+cnNA)Jw&5ftk>a*tpUm4U^h^4AJiY1UZW6?eD1mKA>UnD$zv@RQgKcOW zAR(=^jkTLn>%cZ_j*u{+Z5#{Y9RlTJ=z0tWrsr`IAj3Acl=8NZ9u7145o}|A7)g@X zHUt+~rP+H3*v8yQ65FzE6loqn+nB80J_&4Nbpi>L%Qm{XS)*;7DtLptBe$fGOwlS) z&Z`o0oL@$OZJ1_|nAX`we`<09*}_-*f`crzwz-_G}ix zdmHZEX2il6E0mHDQI&AKpkMqQNqaVn zpkKy){uAG-HjY?xOxsu)#rx0EYl@dUtZn>0m?VkV z#-id2IH!-y_Q$mi!x1E1XX8dm?K!$B}Rpf0S9o4I06xc>TN0KD3Z3r&V zXl%_?u#J)HNo>ouk*ny7wqZ4YnibeanhOb(%QjrX?a(%Q^mM>y$JTBnQ>;ozm6EYa z{8*XR2(}Toi^R0fHZDY#%!2A^zuja#S8=>vGpc8z(?I-5e<|OC&(uG8k@oEBDfkZ^ z-YMWb%mmN;NQl^dQP{=1FFAf2e1K(nVZTz2Mf#4xqMXP34EoZgz404bFJ6%LY!<<+ zW{}ZlJ+Q>QB_X0L(iH6DM^6`Z2Nu=1Pyf5!Jga@GWU%PH(8Yc^AyvU$qs~`Jdp3)p zUuqW~+664RIV41s<&RWkan8Hm04!QtEDmujuRb$atUu@=OUTYuc#7qYy`(*xMbIyc z=4j&E=8FPIh$zdZOUUAuVXOr#m2P*{?lkwy<5~vGwqV}8T&kSC3g(9Ny`(*xrKYLL z1in`;xnG)z{o+ePL|N8eMwW}+Rvw3`5DL3$|e_A#t*7W4<%*928|$zu90L39cmS zyKH07oNXUqZNtrjM2g#n;2f0k|5a0m5%pmo3F1bSz&5-s^iiiuetrEFY-6u432CKm zRNT0x0Jh-~K*EH!(Xfnn(3i@|m}(f9r-Dg<4BPM?5`aTI>+O6Gu#Kn)k|eKf2rh6@ z+uIVbjj|XL+p=xke{lqDBSW==IoL+XMG`2NZ9M;d3ED=4&MUNy;436ktV&3GUcp<~ zwa)!B*hW$&iD{i}m>p5dhU#hguFXcS;@oLqR8Qwh-pX)R?ZUAz);HFW_U!5@nAL*I zZx%xJ?9-<0w~G+FFA6Jp_r?4A@lqFi9bgHyEtqzdd%(K+5NzqwyddNWvJukxM>y|JssCy)7tq&=HO&@bf| zX5jfWk>g2-D9fx2WLcrzDHmpiw#P@z;aH?TN;<+>;qElv5p;SDYWQujk!MJN=yjN0 Z!|V9C%y2c-GxInS)4F=L#Npx4{{u!-Q}O@+ literal 0 HcmV?d00001 diff --git a/logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/trial_1/flags_0.json b/logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/trial_1/flags_0.json new file mode 100644 index 00000000..fea67b90 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/trial_1/flags_0.json @@ -0,0 +1,63 @@ +{ + "logtostderr": false, + "alsologtostderr": false, + "log_dir": "", + "v": 0, + "verbosity": 0, + "logger_levels": {}, + "stderrthreshold": "fatal", + "showprefixforinfo": true, + "run_with_pdb": false, + "pdb_post_mortem": false, + "pdb": false, + "run_with_profiling": false, + "profile_file": null, + "use_cprofile_for_profiling": true, + "only_check_args": false, + "op_conversion_fallback_to_while_loop": true, + "delta_threshold": 0.5, + "tt_check_filter": false, + "tt_single_core_summaries": false, + "runtime_oom_exit": true, + "hbm_oom_exit": true, + "test_srcdir": "", + "test_tmpdir": "/tmp/absl_testing", + "test_random_seed": 301, + "test_randomize_ordering_seed": "", + "xml_output_file": "", + "experimental_orbax_use_distributed_process_id": false, + "experimental_orbax_use_distributed_barrier": false, + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "workload": "imagenet_resnet", + "tuning_ruleset": "self", + "tuning_search_space": null, + "num_tuning_trials": 1, + "data_dir": "/data/imagenet/pytorch", + "imagenet_v2_data_dir": "/data/imagenet/pytorch", + "librispeech_tokenizer_vocab_path": "", + "framework": "pytorch", + "torch_compile": true, + "experiment_dir": "/experiment_runs", + "experiment_name": "submissions_a100/ademamix_golden/study_2", + "save_checkpoints": false, + "save_intermediate_checkpoints": true, + "resume_last_run": null, + "append_timestamp": false, + "use_wandb": false, + "profile": false, + "max_global_steps": null, + "overwrite": true, + "hparam_start_index": null, + "hparam_end_index": null, + "rng_seed": -1587548617, + "set_pytorch_max_split_size": false, + "pytorch_eval_num_workers": 0, + "capture_jax_trace": false, + "skip_evals": false, + "?": false, + "help": false, + "helpshort": false, + "helpfull": false, + "helpxml": false, + "tfds_debug_list_dir": false +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/trial_1/measurements.csv b/logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/trial_1/measurements.csv new file mode 100644 index 00000000..d3db5474 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/trial_1/measurements.csv @@ -0,0 +1,341 @@ +global_step,grad_norm,loss,train/accuracy,train/loss,validation/accuracy,validation/loss,validation/num_examples,test/accuracy,test/loss,test/num_examples,score,total_duration,accumulated_submission_time,accumulated_eval_time,accumulated_logging_time,preemption_count +0,0.4999992251396179,6.922155380249023,,,,,,,,,,,,,, +1,,,0.0010363520408163,6.913094034000319,0.00048,6.91549375,50000.0,0.0011,6.913284375,10000.0,117.69383144378662,306.40399718284607,117.69383144378662,188.0332458019257,0.0,0.0 +1,0.4999992251396179,6.925476551055908,,,,,,,,,,,,,, +2,0.4999992847442627,6.926739692687988,,,,,,,,,,,,,, +3,0.4999993145465851,6.925149440765381,,,,,,,,,,,,,, +4,0.4999992251396179,6.922829627990723,,,,,,,,,,,,,, +5,0.4999992549419403,6.9204583168029785,,,,,,,,,,,,,, +6,0.4999992847442627,6.919160842895508,,,,,,,,,,,,,, +7,0.4999992847442627,6.929003715515137,,,,,,,,,,,,,, +8,0.4999992251396179,6.913826942443848,,,,,,,,,,,,,, +9,0.4999992847442627,6.9154815673828125,,,,,,,,,,,,,, +10,0.4999992549419403,6.934927940368652,,,,,,,,,,,,,, +11,0.4999992847442627,6.933402061462402,,,,,,,,,,,,,, +12,0.4999991953372955,6.926226139068604,,,,,,,,,,,,,, +13,0.4999993145465851,6.926645278930664,,,,,,,,,,,,,, +14,0.4999992549419403,6.933560848236084,,,,,,,,,,,,,, +15,0.4999992549419403,6.924856185913086,,,,,,,,,,,,,, +16,0.4999992847442627,6.932778835296631,,,,,,,,,,,,,, +17,0.4999992251396179,6.917633533477783,,,,,,,,,,,,,, +18,0.4999993145465851,6.921428680419922,,,,,,,,,,,,,, +19,0.4999992251396179,6.920330047607422,,,,,,,,,,,,,, +20,0.4999992549419403,6.925924777984619,,,,,,,,,,,,,, +21,0.4999992549419403,6.9278669357299805,,,,,,,,,,,,,, +22,0.4999992847442627,6.925004959106445,,,,,,,,,,,,,, +23,0.4999992251396179,6.924503803253174,,,,,,,,,,,,,, +24,0.4999992847442627,6.916892528533936,,,,,,,,,,,,,, +25,0.4999992549419403,6.925130844116211,,,,,,,,,,,,,, +26,0.4999992847442627,6.921958923339844,,,,,,,,,,,,,, +27,0.4999992251396179,6.916045188903809,,,,,,,,,,,,,, +28,0.4999992549419403,6.932632446289063,,,,,,,,,,,,,, +29,0.4999991953372955,6.923036575317383,,,,,,,,,,,,,, +30,0.4999991953372955,6.9274444580078125,,,,,,,,,,,,,, +31,0.4999992251396179,6.928824424743652,,,,,,,,,,,,,, +32,0.4999992251396179,6.918056488037109,,,,,,,,,,,,,, +33,0.4999992549419403,6.9180426597595215,,,,,,,,,,,,,, +34,0.4999992549419403,6.92401647567749,,,,,,,,,,,,,, +35,0.4999992847442627,6.926487445831299,,,,,,,,,,,,,, +36,0.4999992847442627,6.92061710357666,,,,,,,,,,,,,, +37,0.4999992847442627,6.922416687011719,,,,,,,,,,,,,, +38,0.4999992549419403,6.911247730255127,,,,,,,,,,,,,, +39,0.4999992251396179,6.928179264068604,,,,,,,,,,,,,, +40,0.4999992847442627,6.922851085662842,,,,,,,,,,,,,, +41,0.4999992847442627,6.920410633087158,,,,,,,,,,,,,, +42,0.4999992251396179,6.915916442871094,,,,,,,,,,,,,, +43,0.4999992549419403,6.9158782958984375,,,,,,,,,,,,,, +44,0.4999992847442627,6.918883800506592,,,,,,,,,,,,,, +45,0.4999992549419403,6.924834251403809,,,,,,,,,,,,,, +46,0.4999992251396179,6.916913032531738,,,,,,,,,,,,,, +47,0.4999992251396179,6.924447059631348,,,,,,,,,,,,,, +48,0.4999992251396179,6.913058280944824,,,,,,,,,,,,,, +49,0.4999992549419403,6.91408634185791,,,,,,,,,,,,,, +50,0.4999991953372955,6.922130584716797,,,,,,,,,,,,,, +51,0.4999992549419403,6.910642623901367,,,,,,,,,,,,,, +52,0.4999992251396179,6.912815093994141,,,,,,,,,,,,,, +53,0.4999992847442627,6.91586446762085,,,,,,,,,,,,,, +54,0.4999992847442627,6.902438163757324,,,,,,,,,,,,,, +55,0.4999992549419403,6.906387805938721,,,,,,,,,,,,,, +56,0.4999992549419403,6.923907279968262,,,,,,,,,,,,,, +57,0.4999992847442627,6.920374870300293,,,,,,,,,,,,,, +58,0.4999992847442627,6.902976036071777,,,,,,,,,,,,,, +59,0.4999992251396179,6.909660339355469,,,,,,,,,,,,,, +60,0.4999992251396179,6.923184394836426,,,,,,,,,,,,,, +61,0.4999992251396179,6.923195838928223,,,,,,,,,,,,,, +62,0.4999992847442627,6.913364410400391,,,,,,,,,,,,,, +63,0.4999992847442627,6.918448448181152,,,,,,,,,,,,,, +64,0.4999992847442627,6.916232109069824,,,,,,,,,,,,,, +65,0.4999992847442627,6.923546314239502,,,,,,,,,,,,,, +66,0.4999992251396179,6.912659645080566,,,,,,,,,,,,,, +67,0.4999992549419403,6.909919261932373,,,,,,,,,,,,,, +68,0.4999992251396179,6.90758228302002,,,,,,,,,,,,,, +69,0.4999992549419403,6.9059247970581055,,,,,,,,,,,,,, +70,0.4999992847442627,6.914321422576904,,,,,,,,,,,,,, +71,0.4999991953372955,6.9033732414245605,,,,,,,,,,,,,, +72,0.4999992549419403,6.908669471740723,,,,,,,,,,,,,, +73,0.4999992549419403,6.911524772644043,,,,,,,,,,,,,, +74,0.4999992549419403,6.913295269012451,,,,,,,,,,,,,, +75,0.4999992251396179,6.888646125793457,,,,,,,,,,,,,, +76,0.4999992251396179,6.912675857543945,,,,,,,,,,,,,, +77,0.4999991953372955,6.903556346893311,,,,,,,,,,,,,, +78,0.4999992847442627,6.898667335510254,,,,,,,,,,,,,, +79,0.4999992549419403,6.907045364379883,,,,,,,,,,,,,, +80,0.4999992549419403,6.905793190002441,,,,,,,,,,,,,, +81,0.4999991655349731,6.904092311859131,,,,,,,,,,,,,, +82,0.4999991953372955,6.896365165710449,,,,,,,,,,,,,, +83,0.4999992251396179,6.90590763092041,,,,,,,,,,,,,, +84,0.4999991953372955,6.897870063781738,,,,,,,,,,,,,, +85,0.4999992847442627,6.895041465759277,,,,,,,,,,,,,, +86,0.4999992251396179,6.893779754638672,,,,,,,,,,,,,, +87,0.4999992847442627,6.903797149658203,,,,,,,,,,,,,, +88,0.4999992847442627,6.893280982971191,,,,,,,,,,,,,, +89,0.4999992847442627,6.905439376831055,,,,,,,,,,,,,, +90,0.4999992549419403,6.901908874511719,,,,,,,,,,,,,, +91,0.4999991953372955,6.898531913757324,,,,,,,,,,,,,, +92,0.4999992251396179,6.904092311859131,,,,,,,,,,,,,, +93,0.4999991953372955,6.897615432739258,,,,,,,,,,,,,, +94,0.4999991953372955,6.894620895385742,,,,,,,,,,,,,, +95,0.4999992847442627,6.899257659912109,,,,,,,,,,,,,, +96,0.4999992847442627,6.904594898223877,,,,,,,,,,,,,, +97,0.4999991953372955,6.896059989929199,,,,,,,,,,,,,, +98,0.4999992847442627,6.892675399780273,,,,,,,,,,,,,, +99,0.4999992549419403,6.902162075042725,,,,,,,,,,,,,, +100,0.4999992251396179,6.893749237060547,,,,,,,,,,,,,, +500,0.4999993741512298,6.4576191902160645,,,,,,,,,,,,,, +1000,0.4999997019767761,5.8594465255737305,,,,,,,,,,,,,, +1500,0.4999998807907104,5.417294979095459,,,,,,,,,,,,,, +2000,0.4999999105930328,5.102451324462891,,,,,,,,,,,,,, +2500,0.4999999105930328,4.762785911560059,,,,,,,,,,,,,, +3000,0.4999999105930328,4.363968849182129,,,,,,,,,,,,,, +3229,,,0.1621293048469387,5.140706276407047,0.15864,4.9367896875,50000.0,0.1165,4.81738046875,10000.0,2108.175397634506,2424.668477535248,2108.175397634506,309.0745980739593,0.031135082244873,0.0 +3500,0.4999999105930328,4.1573405265808105,,,,,,,,,,,,,, +4000,0.4999999105930328,3.881397485733032,,,,,,,,,,,,,, +4500,0.499999850988388,3.519795656204224,,,,,,,,,,,,,, +5000,0.4999999403953552,3.4772868156433105,,,,,,,,,,,,,, +5500,0.499999850988388,3.367765188217163,,,,,,,,,,,,,, +5985,,,0.3155691964285714,4.7227113684829405,0.3118,4.254304375,50000.0,0.2437,3.977594921875,10000.0,4105.261125326157,4564.438182592392,4105.261125326157,447.5696070194245,0.3852415084838867,0.0 +6000,0.4999998807907104,3.0648393630981445,,,,,,,,,,,,,, +6500,0.4999998211860657,2.82855224609375,,,,,,,,,,,,,, +7000,0.4999999105930328,2.8065733909606934,,,,,,,,,,,,,, +7500,0.499999850988388,2.7629318237304688,,,,,,,,,,,,,, +8000,0.4999999105930328,2.5699245929718018,,,,,,,,,,,,,, +8500,0.499999850988388,2.4263148307800293,,,,,,,,,,,,,, +8666,,,0.4215760522959184,4.734850513691804,0.4368,3.561740625,50000.0,0.3598,3.02988828125,10000.0,6103.339204072952,6707.132169008255,6103.339204072952,588.3147366046906,0.769068717956543,0.0 +9000,0.4999999105930328,2.350368022918701,,,,,,,,,,,,,, +9500,0.4999999105930328,2.4039952754974365,,,,,,,,,,,,,, +10000,0.4999998211860657,2.132540941238404,,,,,,,,,,,,,, +10500,0.499999850988388,2.2133612632751465,,,,,,,,,,,,,, +11000,0.4999997913837433,2.046342372894287,,,,,,,,,,,,,, +11342,,,0.519172512755102,3.8327312858737246,0.49202,3.432515625,50000.0,0.419,2.7275841796875,10000.0,8096.436839818954,8822.90674829483,8096.436839818954,706.2089529037476,1.0964832305908203,0.0 +11500,0.4999998211860657,1.9242291450500488,,,,,,,,,,,,,, +12000,0.4999997615814209,1.9483561515808103,,,,,,,,,,,,,, +12500,0.4999997913837433,2.0536694526672363,,,,,,,,,,,,,, +13000,0.4999998211860657,1.9311683177948,,,,,,,,,,,,,, +13500,0.4999998807907104,1.8908159732818604,,,,,,,,,,,,,, +14000,0.4999997913837433,1.855334758758545,,,,,,,,,,,,,, +14006,,,0.5588329081632653,3.752557248485332,0.52142,3.3949671875,50000.0,0.4528,2.601360546875,10000.0,10089.947731256483,10943.603086471558,10089.947731256483,829.4630072116852,1.425502061843872,0.0 +14500,0.499999850988388,1.854910135269165,,,,,,,,,,,,,, +15000,0.4999996721744537,1.813289284706116,,,,,,,,,,,,,, +15500,0.4999997913837433,1.6871652603149414,,,,,,,,,,,,,, +16000,0.4999997615814209,1.7803120613098145,,,,,,,,,,,,,, +16500,0.4999997317790985,1.6718060970306396,,,,,,,,,,,,,, +16644,,,0.5981146364795918,3.628994377291933,0.54502,3.5477440625,50000.0,0.4807,2.3738681640625,10000.0,12082.892327547072,13070.141052007675,12082.892327547072,958.7037978172302,1.7975919246673584,0.0 +17000,0.4999997019767761,1.702135443687439,,,,,,,,,,,,,, +17500,0.4999997019767761,1.7457648515701294,,,,,,,,,,,,,, +18000,0.4999997615814209,1.689683198928833,,,,,,,,,,,,,, +18500,0.4999997615814209,1.7048873901367188,,,,,,,,,,,,,, +19000,0.4999997615814209,1.5879864692687988,,,,,,,,,,,,,, +19287,,,0.5989516900510204,3.800162179129464,0.55098,3.3991665625,50000.0,0.4909,2.3680232421875,10000.0,14081.901285886765,15199.4295668602,14081.901285886765,1083.2835838794708,2.191770076751709,0.0 +19500,0.4999997317790985,1.5322413444519043,,,,,,,,,,,,,, +20000,0.4999997615814209,1.5941097736358645,,,,,,,,,,,,,, +20500,0.4999998211860657,1.62510347366333,,,,,,,,,,,,,, +21000,0.4999996721744537,1.481923222541809,,,,,,,,,,,,,, +21500,0.4999997913837433,1.5587399005889893,,,,,,,,,,,,,, +21970,,,0.5838647959183674,4.3787888507453765,0.56536,3.209695625,50000.0,0.4916,2.3627771484375,10000.0,16076.210381507874,17326.093100070953,16076.210381507874,1209.900592803955,2.501425504684448,0.0 +22000,0.4999998807907104,1.5449843406677246,,,,,,,,,,,,,, +22500,0.4999996721744537,1.5085383653640747,,,,,,,,,,,,,, +23000,0.4999996721744537,1.4754573106765747,,,,,,,,,,,,,, +23500,0.4999997317790985,1.4101269245147705,,,,,,,,,,,,,, +24000,0.4999996721744537,1.4530627727508545,,,,,,,,,,,,,, +24500,0.4999997019767761,1.487351894378662,,,,,,,,,,,,,, +24669,,,0.640625,3.562229078643176,0.56716,3.4612678125,50000.0,0.5159,2.26638359375,10000.0,18070.13053560257,19468.0190050602,18070.13053560257,1352.7498168945312,2.8937878608703613,0.0 +25000,0.4999996721744537,1.5035879611968994,,,,,,,,,,,,,, +25500,0.4999997019767761,1.462825059890747,,,,,,,,,,,,,, +26000,0.4999996721744537,1.4062477350234983,,,,,,,,,,,,,, +26500,0.4999996423721313,1.4062217473983765,,,,,,,,,,,,,, +27000,0.4999997615814209,1.43276047706604,,,,,,,,,,,,,, +27331,,,0.6507095025510204,3.3582365074936225,0.5733,3.3512609375,50000.0,0.5157,2.247359765625,10000.0,20065.60669398308,21602.546391010284,20065.60669398308,1487.866794347763,3.247710704803467,0.0 +27500,0.4999997615814209,1.4082765579223633,,,,,,,,,,,,,, +28000,0.4999996125698089,1.2943248748779297,,,,,,,,,,,,,, +28500,0.4999997317790985,1.4253463745117188,,,,,,,,,,,,,, +29000,0.4999997317790985,1.2439850568771362,,,,,,,,,,,,,, +29500,0.4999997019767761,1.3386977910995483,,,,,,,,,,,,,, +29969,,,0.6632852359693877,3.6978236607142856,0.59528,3.07289625,50000.0,0.5312,2.207117578125,10000.0,22061.40389227867,23713.798048257828,22061.40389227867,1599.0447170734406,3.656982898712158,0.0 +30000,0.4999997615814209,1.2664144039154053,,,,,,,,,,,,,, +30500,0.4999996125698089,1.31471586227417,,,,,,,,,,,,,, +31000,0.4999995827674866,1.2223153114318848,,,,,,,,,,,,,, +31500,0.4999996423721313,1.2160015106201172,,,,,,,,,,,,,, +32000,0.4999996721744537,1.2996199131011963,,,,,,,,,,,,,, +32500,0.4999996423721313,1.472954511642456,,,,,,,,,,,,,, +32527,,,0.6575454400510204,4.162380296356824,0.58764,3.505524375,50000.0,0.5319,2.2375390625,10000.0,24056.594072580338,25833.816573143005,24056.594072580338,1719.7836360931396,4.115094661712647,0.0 +33000,0.4999997019767761,1.2763121128082275,,,,,,,,,,,,,, +33500,0.4999997615814209,1.2878811359405518,,,,,,,,,,,,,, +34000,0.4999995827674866,1.1777909994125366,,,,,,,,,,,,,, +34500,0.4999996125698089,1.2829058170318604,,,,,,,,,,,,,, +35000,0.4999997019767761,1.1536900997161863,,,,,,,,,,,,,, +35088,,,0.6999362244897959,3.3552706971460458,0.59926,3.23747375,50000.0,0.5413,2.1987033203125,10000.0,26049.596282482147,27952.762268304825,26049.596282482147,1841.544192790985,4.55816388130188,0.0 +35500,0.4999996423721313,1.2322622537612915,,,,,,,,,,,,,, +36000,0.4999996423721313,1.1895064115524292,,,,,,,,,,,,,, +36500,0.4999996721744537,1.1960045099258425,,,,,,,,,,,,,, +37000,0.4999995529651642,1.223503351211548,,,,,,,,,,,,,, +37500,0.4999996125698089,1.1863670349121094,,,,,,,,,,,,,, +37681,,,0.7167171556122449,3.406367710658482,0.60236,3.4714803125,50000.0,0.527,2.26468828125,10000.0,28042.45645093918,30073.98717403412,28042.45645093918,1965.5194764137268,5.167999982833862,0.0 +38000,0.4999995231628418,1.1012616157531738,,,,,,,,,,,,,, +38500,0.4999996423721313,1.18671977519989,,,,,,,,,,,,,, +39000,0.4999996721744537,1.1104694604873655,,,,,,,,,,,,,, +39500,0.4999995529651642,1.1719164848327637,,,,,,,,,,,,,, +40000,0.4999995529651642,1.2869060039520264,,,,,,,,,,,,,, +40351,,,0.7042012117346939,3.533531033262914,0.58318,3.826075,50000.0,0.5265,2.308673828125,10000.0,30035.08023881912,32200.80206418037,30035.08023881912,2095.101897716522,5.541431188583374,0.0 +40500,0.4999997615814209,1.2224746942520142,,,,,,,,,,,,,, +41000,0.4999996423721313,1.1457881927490234,,,,,,,,,,,,,, +41500,0.4999996423721313,1.2025954723358154,,,,,,,,,,,,,, +42000,0.4999996423721313,1.1436655521392822,,,,,,,,,,,,,, +42500,0.4999997019767761,1.117009520530701,,,,,,,,,,,,,, +42975,,,0.7175143494897959,3.674822748923788,0.60794,3.3489609375,50000.0,0.5359,2.2669248046875,10000.0,32028.396780967712,34328.45838928223,32028.396780967712,2225.492896080017,5.898090839385986,0.0 +43000,0.4999996423721313,1.2544173002243042,,,,,,,,,,,,,, +43500,0.4999995231628418,1.1540741920471191,,,,,,,,,,,,,, +44000,0.4999996423721313,1.0613608360290527,,,,,,,,,,,,,, +44500,0.4999997019767761,1.0333502292633057,,,,,,,,,,,,,, +45000,0.4999995231628418,1.033962965011597,,,,,,,,,,,,,, +45500,0.4999996423721313,1.0428059101104736,,,,,,,,,,,,,, +45639,,,0.7144650829081632,3.772730924645249,0.60726,3.3839653125,50000.0,0.5427,2.2691771484375,10000.0,34023.23478484154,36454.61201810837,34023.23478484154,2350.798717737198,6.316791296005249,0.0 +46000,0.4999995827674866,1.1782243251800537,,,,,,,,,,,,,, +46500,0.4999996125698089,1.1152465343475342,,,,,,,,,,,,,, +47000,0.4999997019767761,1.118058681488037,,,,,,,,,,,,,, +47500,0.4999995827674866,1.068516492843628,,,,,,,,,,,,,, +48000,0.4999996721744537,1.0129449367523191,,,,,,,,,,,,,, +48310,,,0.7372249681122449,3.4263698032924106,0.59002,3.86083,50000.0,0.5378,2.2942724609375,10000.0,36016.49505734444,38586.453929424286,36016.49505734444,2483.848254919052,6.711942195892334,0.0 +48500,0.4999995529651642,0.9911210536956788,,,,,,,,,,,,,, +49000,0.4999997317790985,1.026638150215149,,,,,,,,,,,,,, +49500,0.4999996721744537,1.0645451545715332,,,,,,,,,,,,,, +50000,0.4999996721744537,1.0450539588928225,,,,,,,,,,,,,, +50500,0.4999995827674866,0.9529215097427368,,,,,,,,,,,,,, +50983,,,0.7030253507653061,4.454295878507653,0.60278,3.635655625,50000.0,0.5423,2.321203125,10000.0,38012.98178291321,40719.979044914246,38012.98178291321,2615.5865807533264,7.114745140075684,0.0 +51000,0.4999996423721313,1.03725004196167,,,,,,,,,,,,,, +51500,0.4999995827674866,0.9771136045455932,,,,,,,,,,,,,, +52000,0.4999995827674866,1.0761849880218506,,,,,,,,,,,,,, +52500,0.4999995827674866,1.0842736959457395,,,,,,,,,,,,,, +53000,0.4999996125698089,0.9404076337814332,,,,,,,,,,,,,, +53500,0.4999995827674866,1.0349645614624023,,,,,,,,,,,,,, +53643,,,0.7354512117346939,3.9732946279097576,0.60552,3.769000625,50000.0,0.5485,2.3113056640625,10000.0,40020.47555828095,42867.38005900383,40020.47555828095,2751.5167932510376,7.502183437347412,0.0 +54000,0.4999995827674866,1.1073195934295654,,,,,,,,,,,,,, +54500,0.4999995827674866,1.0206149816513062,,,,,,,,,,,,,, +55000,0.4999995231628418,1.0596911907196045,,,,,,,,,,,,,, +55500,0.4999995529651642,0.9092148542404176,,,,,,,,,,,,,, +56000,0.4999996125698089,0.90851891040802,,,,,,,,,,,,,, +56265,,,0.7109375,4.522355994399713,0.60638,3.6884934375,50000.0,0.5361,2.37835546875,10000.0,42013.84645605087,44960.60993313789,42013.84645605087,2847.636313676834,7.968269109725952,0.0 +56500,0.4999995529651642,0.8705559968948364,,,,,,,,,,,,,, +57000,0.4999996125698089,0.8993426561355591,,,,,,,,,,,,,, +57500,0.4999996423721313,0.9842264652252196,,,,,,,,,,,,,, +58000,0.4999995827674866,0.9017523527145386,,,,,,,,,,,,,, +58500,0.4999996423721313,1.0559101104736328,,,,,,,,,,,,,, +58754,,,0.7159996811224489,4.573967135682398,0.5886,4.122043125,50000.0,0.5422,2.350933203125,10000.0,44008.71176171303,47062.44623923302,44008.71176171303,2950.9123871326447,8.387396335601807,0.0 +59000,0.4999996125698089,0.992040753364563,,,,,,,,,,,,,, +59500,0.4999997019767761,0.9364137053489684,,,,,,,,,,,,,, +60000,0.4999995827674866,1.021019697189331,,,,,,,,,,,,,, +60500,0.4999996125698089,0.9992450475692748,,,,,,,,,,,,,, +61000,0.4999995529651642,0.9671766757965088,,,,,,,,,,,,,, +61268,,,0.7661830357142857,3.472351697026467,0.59942,4.078488125,50000.0,0.5413,2.3884078125,10000.0,46006.77871727944,49160.60372567177,46006.77871727944,3047.156206846237,8.810739994049072,0.0 +61500,0.4999995827674866,0.8322563171386719,,,,,,,,,,,,,, +62000,0.4999995231628418,0.9231167435646056,,,,,,,,,,,,,, +62500,0.4999997019767761,1.0612024068832395,,,,,,,,,,,,,, +63000,0.4999995529651642,0.8733479976654053,,,,,,,,,,,,,, +63500,0.4999995529651642,0.8659032583236694,,,,,,,,,,,,,, +63810,,,0.7591677295918368,3.811367112763074,0.59584,4.0934078125,50000.0,0.5409,2.397580859375,10000.0,47999.9391438961,51283.937019109726,47999.9391438961,3172.961097240448,9.259364604949951,0.0 +64000,0.4999996125698089,0.949940025806427,,,,,,,,,,,,,, +64500,0.4999996125698089,0.8146262168884277,,,,,,,,,,,,,, +65000,0.4999995827674866,1.0230642557144165,,,,,,,,,,,,,, +65500,0.4999995529651642,0.836719274520874,,,,,,,,,,,,,, +66000,0.4999995827674866,1.1181788444519043,,,,,,,,,,,,,, +66364,,,0.7644491390306123,3.9005092698700574,0.6022,3.884179375,50000.0,0.5409,2.4544275390625,10000.0,49994.2833840847,53406.36391091347,49994.2833840847,3296.469463825226,9.629589080810549,0.0 +66500,0.4999995529651642,0.8400009870529175,,,,,,,,,,,,,, +67000,0.4999996721744537,0.8982428312301636,,,,,,,,,,,,,, +67500,0.4999996125698089,0.9567756056785583,,,,,,,,,,,,,, +68000,0.4999995231628418,0.8255354762077332,,,,,,,,,,,,,, +68500,0.4999996125698089,0.8403891324996948,,,,,,,,,,,,,, +68925,,,0.7666414221938775,3.916106554926658,0.60528,3.85621125,50000.0,0.5375,2.504280078125,10000.0,51988.39248514176,55528.33761072159,51988.39248514176,3419.99373793602,10.09680700302124,0.0 +69000,0.4999996423721313,0.8710421323776245,,,,,,,,,,,,,, +69500,0.4999995827674866,0.8670406341552734,,,,,,,,,,,,,, +70000,0.4999995529651642,0.8900437951087952,,,,,,,,,,,,,, +70500,0.4999996125698089,0.902928590774536,,,,,,,,,,,,,, +71000,0.4999996423721313,0.8306645154953003,,,,,,,,,,,,,, +71500,0.4999995529651642,0.8385712504386902,,,,,,,,,,,,,, +71567,,,0.7959582270408163,3.3684337382413903,0.61068,3.876725625,50000.0,0.5439,2.482401171875,10000.0,53983.56141471863,57650.36901497841,53983.56141471863,3542.36199259758,10.43744444847107,0.0 +72000,0.4999996125698089,0.7808813452720642,,,,,,,,,,,,,, +72500,0.4999996125698089,0.9155505895614624,,,,,,,,,,,,,, +73000,0.4999996423721313,0.891876757144928,,,,,,,,,,,,,, +73500,0.4999995827674866,0.8636876940727234,,,,,,,,,,,,,, +74000,0.4999995529651642,0.7728566527366638,,,,,,,,,,,,,, +74216,,,0.7823262117346939,3.757672368263712,0.60554,4.1128596875,50000.0,0.5405,2.5159078125,10000.0,55976.11931800842,59776.55257821083,55976.11931800842,3671.0648312568665,10.87571096420288,0.0 +74500,0.4999996423721313,0.8381816744804382,,,,,,,,,,,,,, +75000,0.4999996125698089,0.8950157165527344,,,,,,,,,,,,,, +75500,0.4999996125698089,0.7749454975128174,,,,,,,,,,,,,, +76000,0.4999995827674866,0.810245156288147,,,,,,,,,,,,,, +76500,0.4999995231628418,0.7637450695037842,,,,,,,,,,,,,, +76891,,,0.7877471301020408,3.733782709861288,0.60232,4.02851125,50000.0,0.5366,2.53688671875,10000.0,57973.79527235031,61914.58137059212,57973.79527235031,3806.97487950325,11.273123502731323,0.0 +77000,0.4999996423721313,0.8542247414588928,,,,,,,,,,,,,, +77500,0.4999996423721313,0.8631196618080139,,,,,,,,,,,,,, +78000,0.4999995529651642,0.8193002343177795,,,,,,,,,,,,,, +78500,0.4999996125698089,0.8018485307693481,,,,,,,,,,,,,, +79000,0.4999996125698089,0.7915757894515991,,,,,,,,,,,,,, +79500,0.4999996125698089,0.8119314908981323,,,,,,,,,,,,,, +79571,,,0.7647082270408163,4.528148417570153,0.59836,4.343521875,50000.0,0.542,2.5561978515625,10000.0,59967.32413458824,64057.164492845535,59967.32413458824,3950.907996416092,11.60761857032776,0.0 +80000,0.4999996423721313,0.9174575805664062,,,,,,,,,,,,,, +80500,0.4999995827674866,0.8162398934364319,,,,,,,,,,,,,, +81000,0.4999995827674866,0.880406379699707,,,,,,,,,,,,,, +81500,0.4999996125698089,0.7729280591011047,,,,,,,,,,,,,, +82000,0.4999996721744537,0.8311651945114136,,,,,,,,,,,,,, +82235,,,0.7634526466836735,4.757924916792889,0.60174,4.3246,50000.0,0.5413,2.5651658203125,10000.0,61964.11759972572,66187.10884904861,61964.11759972572,4078.8110699653625,11.951503276824951,0.0 +82500,0.4999995231628418,0.7348687648773193,,,,,,,,,,,,,, +83000,0.4999995827674866,0.8217381238937378,,,,,,,,,,,,,, +83500,0.4999996423721313,0.8173751831054688,,,,,,,,,,,,,, +84000,0.4999996423721313,0.7956050634384155,,,,,,,,,,,,,, +84500,0.4999995231628418,0.8711147308349609,,,,,,,,,,,,,, +84895,,,0.7957589285714286,3.9244023537149233,0.60644,4.156065625,50000.0,0.5449,2.5498849609375,10000.0,63957.24264502525,68333.47369885445,63957.24264502525,4228.070791721344,12.361259937286375,0.0 +85000,0.4999996125698089,0.8944044709205627,,,,,,,,,,,,,, +85500,0.4999996125698089,0.7455011606216431,,,,,,,,,,,,,, +86000,0.4999996125698089,0.8501022458076477,,,,,,,,,,,,,, +86500,0.4999996423721313,0.7588382959365845,,,,,,,,,,,,,, +87000,0.4999996125698089,0.7324253916740417,,,,,,,,,,,,,, +87500,0.4999995827674866,0.7609399557113647,,,,,,,,,,,,,, +87519,,,0.7683952487244898,4.895434009785554,0.60802,4.351689375,50000.0,0.5401,2.64440703125,10000.0,65951.39333939552,70437.01794242859,65951.39333939552,4333.659548521042,12.685445070266724,0.0 +88000,0.4999995529651642,0.7027808427810669,,,,,,,,,,,,,, +88500,0.4999996125698089,0.8223464488983154,,,,,,,,,,,,,, +89000,0.4999995529651642,0.6864622831344604,,,,,,,,,,,,,, +89500,0.4999996125698089,0.727545976638794,,,,,,,,,,,,,, +90000,0.4999995827674866,0.7946692705154419,,,,,,,,,,,,,, +90073,,,0.7809709821428571,4.373063690808355,0.61414,3.89102,50000.0,0.5335,2.604086328125,10000.0,67952.03266215324,72562.04839730263,67952.03266215324,4454.272779941559,13.008050680160522,0.0 +90500,0.4999995827674866,0.7456266283988953,,,,,,,,,,,,,, +91000,0.4999996721744537,0.7781481742858887,,,,,,,,,,,,,, +91500,0.4999995529651642,0.6985890865325928,,,,,,,,,,,,,, +92000,0.4999995827674866,0.7044679522514343,,,,,,,,,,,,,, +92500,0.4999996423721313,0.6825626492500305,,,,,,,,,,,,,, +92626,,,0.7737364477040817,4.867963518415179,0.59746,4.616288125,50000.0,0.538,2.6560337890625,10000.0,69945.19483971596,74684.95734524727,69945.19483971596,4579.605948209763,13.406023025512695,0.0 +93000,0.4999996721744537,0.7330970764160156,,,,,,,,,,,,,, +93500,0.4999995827674866,0.7487102150917053,,,,,,,,,,,,,, +94000,0.4999996125698089,0.6944018006324768,,,,,,,,,,,,,, +94500,0.4999995231628418,0.753358006477356,,,,,,,,,,,,,, +95000,0.4999996125698089,0.7932641506195068,,,,,,,,,,,,,, +95245,,,0.7448182397959183,5.682906793088329,0.61956,3.79448125,50000.0,0.546,2.6514984375,10000.0,71939.20494151115,76808.11458563805,71939.20494151115,4704.600276708603,13.790885210037231,0.0 +95500,0.4999995529651642,0.6461749076843262,,,,,,,,,,,,,, +96000,0.4999995231628418,0.5890814661979675,,,,,,,,,,,,,, +96500,0.4999995529651642,0.6714351773262024,,,,,,,,,,,,,, +97000,0.4999995231628418,0.6677297353744507,,,,,,,,,,,,,, +97500,0.4999997615814209,0.7446516752243042,,,,,,,,,,,,,, +97839,,,0.7731784119897959,4.878094731544961,0.61604,4.014446875,50000.0,0.5366,2.770053125,10000.0,73931.97438669205,78929.10322093964,73931.97438669205,4828.478740930557,14.217012643814089,0.0 +98000,0.4999995529651642,0.683371901512146,,,,,,,,,,,,,, +98500,0.4999995529651642,0.7079941630363464,,,,,,,,,,,,,, +99000,0.4999995231628418,0.6334831714630127,,,,,,,,,,,,,, +99500,0.4999995827674866,0.7113249897956848,,,,,,,,,,,,,, +100000,0.4999995231628418,0.706080436706543,,,,,,,,,,,,,, +100480,,,,,,,,,,,75927.63577508926,,,,,0.0 diff --git a/logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/trial_1/meta_data_0.json b/logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/trial_1/meta_data_0.json new file mode 100644 index 00000000..adac067b --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/imagenet_resnet_pytorch/trial_1/meta_data_0.json @@ -0,0 +1,71 @@ +{ + "workload.bn_init_scale": 0.0, + "workload.center_crop_size": 224, + "workload.eval_batch_size": 1024, + "workload.eval_num_workers": 0, + "workload.eval_period_time_sec": 1996, + "workload.max_allowed_runtime_sec": 49918, + "workload.num_eval_train_examples": 50176, + "workload.num_test_examples": 10000, + "workload.num_train_examples": 1281167, + "workload.num_validation_examples": 50000, + "workload.resize_size": 256, + "workload.step_hint": 195999, + "workload.target_metric_name": "accuracy", + "workload.test_target_value": 0.656, + "workload.use_gelu": false, + "workload.use_silu": false, + "workload.validation_target_value": 0.77431, + "cpu.util.avg_percent_since_last": 9.3, + "cpu.freq.current": 2200.1639999999993, + "mem.total": 359053524992, + "mem.available": 347651780608, + "mem.used": 8245927936, + "mem.percent_used": 3.2, + "mem.read_bytes_since_boot": 7811987540992, + "mem.write_bytes_since_boot": 66015220736, + "net.bytes_sent_since_boot": 31298137, + "net.bytes_recv_since_boot": 31301169, + "gpu.count": 4, + "gpu.0.compute.util": 0.0, + "gpu.0.mem.util": 0.0322998046875, + "gpu.0.mem.total": 40960.0, + "gpu.0.mem.used": 1323.0, + "gpu.0.mem.free": 39004.0, + "gpu.0.temp.current": 34.0, + "gpu.1.compute.util": 0.0, + "gpu.1.mem.util": 0.0322998046875, + "gpu.1.mem.total": 40960.0, + "gpu.1.mem.used": 1323.0, + "gpu.1.mem.free": 39004.0, + "gpu.1.temp.current": 37.0, + "gpu.2.compute.util": 0.0, + "gpu.2.mem.util": 0.0322998046875, + "gpu.2.mem.total": 40960.0, + "gpu.2.mem.used": 1323.0, + "gpu.2.mem.free": 39004.0, + "gpu.2.temp.current": 34.0, + "gpu.3.compute.util": 0.0, + "gpu.3.mem.util": 0.0322998046875, + "gpu.3.mem.total": 40960.0, + "gpu.3.mem.used": 1323.0, + "gpu.3.mem.free": 39004.0, + "gpu.3.temp.current": 36.0, + "gpu.avg.compute.util": 0.0, + "gpu.avg.mem.util": 0.0322998046875, + "gpu.avg.mem.total": 40960.0, + "gpu.avg.mem.used": 1323.0, + "gpu.avg.mem.free": 39004.0, + "gpu.avg.temp.current": 35.25, + "os_platform": "Linux-6.1.0-44-cloud-amd64-x86_64-with-glibc2.31", + "python_version": "3.11.10", + "python_compiler": "GCC 9.4.0", + "git_branch": "main", + "git_commit_hash": "b21be29be0a1573fb4f78f849aea019cdb520862", + "cpu_model_name": "Intel(R) Xeon(R) CPU @ 2.20GHz", + "cpu_count": 24, + "gpu_model_name": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_driver": "550.90.12", + "rng_seed": -1587548617 +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/imagenet_vit_pytorch_09-15-2026-10-03-35.log b/logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/imagenet_vit_pytorch_09-15-2026-10-03-35.log new file mode 100644 index 00000000..e2f6897d --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/imagenet_vit_pytorch_09-15-2026-10-03-35.log @@ -0,0 +1,1079 @@ +torchrun --redirects 1:0,2:0,3:0 --standalone --nnodes=1 --nproc_per_node=4 submission_runner.py --framework=pytorch --workload=imagenet_vit --submission_path=submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py --data_dir=/data/imagenet/pytorch --experiment_dir=/experiment_runs --experiment_name=submissions_a100/ademamix_golden/study_2 --overwrite=True --save_checkpoints=False --rng_seed=-83011591 --imagenet_v2_data_dir=/data/imagenet/pytorch --torch_compile=true --tuning_ruleset=self 2>&1 | tee -a /logs/imagenet_vit_pytorch_09-15-2026-10-03-35.log +W0915 10:03:36.777000 9 site-packages/torch/distributed/run.py:803] +W0915 10:03:36.777000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0915 10:03:36.777000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0915 10:03:36.777000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-15 10:03:38.256595: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-15 10:03:38.256595: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-15 10:03:38.256636: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-15 10:03:38.256898: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789466618.281143 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789466618.281124 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789466618.281124 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789466618.281125 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789466618.289127 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789466618.289129 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789466618.289127 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789466618.289129 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789466618.315559 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789466618.315559 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789466618.315559 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789466618.315559 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789466618.315588 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789466618.315588 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789466618.315589 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789466618.315591 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789466618.315591 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789466618.315591 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789466618.315593 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789466618.315593 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789466618.315593 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789466618.315594 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789466618.315596 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789466618.315598 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789466624.048957 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789466624.381930 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789466624.453678 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789466624.506958 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W915 10:03:49.775062069 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0915 10:03:49.786661 140652717921472 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/imagenet_vit_pytorch. +I0915 10:03:49.786661 139691287286976 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/imagenet_vit_pytorch. +I0915 10:03:49.786661 140633339442368 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/imagenet_vit_pytorch. +I0915 10:03:49.786700 140042512016576 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/imagenet_vit_pytorch. +I0915 10:03:49.810455 139691287286976 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/imagenet_vit_pytorch/trial_1. +I0915 10:03:50.142045 139691287286976 submission_runner.py:242] Initializing dataset. +I0915 10:04:20.350748 139691287286976 submission_runner.py:251] Initializing model. +I0915 10:04:20.723613 139691287286976 submission_runner.py:290] Performing `torch.compile`. +I0915 10:04:21.790481 139691287286976 submission_runner.py:294] Initializing optimizer. +I0915 10:04:21.791396 139691287286976 submission_runner.py:299] Initializing metrics bundle. +I0915 10:04:21.791550 139691287286976 submission_runner.py:321] Initializing checkpoint and logger. +I0915 10:04:21.792900 139691287286976 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_2/imagenet_vit_pytorch/trial_1/meta_data_0.json. +I0915 10:04:22.005602 139691287286976 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_2/imagenet_vit_pytorch/trial_1/flags_0.json. +I0915 10:04:22.047821 139691287286976 submission_runner.py:359] Starting training loop. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +[rank0]:W0915 10:04:47.947000 38 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank1]:W0915 10:04:49.152000 39 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank2]:W0915 10:04:49.370000 40 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank3]:W0915 10:04:49.518000 41 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +I0915 10:06:23.166832 139666346989312 logging_writer.py:48] [0] global_step=0, grad_norm=0.355771, loss=6.90776 +I0915 10:06:23.371384 139691287286976 submission.py:307] 0) loss = 6.908, grad_norm = 0.356 +I0915 10:06:24.105358 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 10:08:43.142538 139691287286976 spec.py:346] Evaluating on the validation split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 10:09:52.200263 139691287286976 spec.py:363] Evaluating on the test split. +I0915 10:09:53.523038 139691287286976 dataset_info.py:707] Load dataset info from /data/imagenet/pytorch/imagenet_v2/matched-frequency/3.0.0 +I0915 10:09:53.583471 139691287286976 reader.py:262] Creating a tf.data.Dataset reading 16 files located in folders: /data/imagenet/pytorch/imagenet_v2/matched-frequency/3.0.0. +I0915 10:09:53.866560 139691287286976 logging_logger.py:49] Constructing tf.data.Dataset imagenet_v2 for split test, from /data/imagenet/pytorch/imagenet_v2/matched-frequency/3.0.0 +I0915 10:10:51.006578 139691287286976 submission_runner.py:516] Time since start: 388.96s, Step: 1, {'train/accuracy': 0.00228515625, 'train/loss': 6.90775634765625, 'validation/accuracy': 0.00176, 'validation/loss': 6.90775625, 'validation/num_examples': 50000, 'test/accuracy': 0.0013, 'test/loss': 6.90775546875, 'test/num_examples': 10000, 'score': 121.32478308677673, 'total_duration': 388.9585015773773, 'accumulated_submission_time': 121.32478308677673, 'accumulated_eval_time': 266.90091943740845, 'accumulated_logging_time': 0} +I0915 10:10:51.214229 139636869428992 logging_writer.py:48] [1] accumulated_eval_time=266.901, accumulated_logging_time=0, accumulated_submission_time=121.325, global_step=1, preemption_count=0, score=121.325, test/accuracy=0.0013, test/loss=6.90776, test/num_examples=10000, total_duration=388.959, train/accuracy=0.00228516, train/loss=6.90776, validation/accuracy=0.00176, validation/loss=6.90776, validation/num_examples=50000 +I0915 10:10:52.607192 139636861036288 logging_writer.py:48] [1] global_step=1, grad_norm=0.372583, loss=6.90776 +I0915 10:10:52.610348 139691287286976 submission.py:307] 1) loss = 6.908, grad_norm = 0.373 +I0915 10:10:52.853782 139636869428992 logging_writer.py:48] [2] global_step=2, grad_norm=0.384194, loss=6.90775 +I0915 10:10:52.856992 139691287286976 submission.py:307] 2) loss = 6.908, grad_norm = 0.384 +I0915 10:10:53.137900 139636861036288 logging_writer.py:48] [3] global_step=3, grad_norm=0.370934, loss=6.90775 +I0915 10:10:53.143202 139691287286976 submission.py:307] 3) loss = 6.908, grad_norm = 0.371 +I0915 10:10:53.369396 139636869428992 logging_writer.py:48] [4] global_step=4, grad_norm=0.366719, loss=6.90775 +I0915 10:10:53.373034 139691287286976 submission.py:307] 4) loss = 6.908, grad_norm = 0.367 +I0915 10:10:53.608224 139636861036288 logging_writer.py:48] [5] global_step=5, grad_norm=0.372001, loss=6.90776 +I0915 10:10:53.613502 139691287286976 submission.py:307] 5) loss = 6.908, grad_norm = 0.372 +I0915 10:10:53.851601 139636869428992 logging_writer.py:48] [6] global_step=6, grad_norm=0.381048, loss=6.90775 +I0915 10:10:53.855516 139691287286976 submission.py:307] 6) loss = 6.908, grad_norm = 0.381 +I0915 10:10:54.072277 139636861036288 logging_writer.py:48] [7] global_step=7, grad_norm=0.373081, loss=6.90775 +I0915 10:10:54.075582 139691287286976 submission.py:307] 7) loss = 6.908, grad_norm = 0.373 +I0915 10:10:54.328791 139636869428992 logging_writer.py:48] [8] global_step=8, grad_norm=0.366912, loss=6.90775 +I0915 10:10:54.332634 139691287286976 submission.py:307] 8) loss = 6.908, grad_norm = 0.367 +I0915 10:10:54.557727 139636861036288 logging_writer.py:48] [9] global_step=9, grad_norm=0.369815, loss=6.90775 +I0915 10:10:54.561085 139691287286976 submission.py:307] 9) loss = 6.908, grad_norm = 0.370 +I0915 10:10:54.787293 139636869428992 logging_writer.py:48] [10] global_step=10, grad_norm=0.382893, loss=6.90773 +I0915 10:10:54.790517 139691287286976 submission.py:307] 10) loss = 6.908, grad_norm = 0.383 +I0915 10:10:55.027123 139636861036288 logging_writer.py:48] [11] global_step=11, grad_norm=0.361936, loss=6.90773 +I0915 10:10:55.031357 139691287286976 submission.py:307] 11) loss = 6.908, grad_norm = 0.362 +I0915 10:10:55.251970 139636869428992 logging_writer.py:48] [12] global_step=12, grad_norm=0.372869, loss=6.90774 +I0915 10:10:55.255452 139691287286976 submission.py:307] 12) loss = 6.908, grad_norm = 0.373 +I0915 10:10:55.473790 139636861036288 logging_writer.py:48] [13] global_step=13, grad_norm=0.372539, loss=6.90773 +I0915 10:10:55.477120 139691287286976 submission.py:307] 13) loss = 6.908, grad_norm = 0.373 +I0915 10:10:55.701698 139636869428992 logging_writer.py:48] [14] global_step=14, grad_norm=0.368976, loss=6.90775 +I0915 10:10:55.704685 139691287286976 submission.py:307] 14) loss = 6.908, grad_norm = 0.369 +I0915 10:10:55.935691 139636861036288 logging_writer.py:48] [15] global_step=15, grad_norm=0.374493, loss=6.90772 +I0915 10:10:55.939882 139691287286976 submission.py:307] 15) loss = 6.908, grad_norm = 0.374 +I0915 10:10:56.168128 139636869428992 logging_writer.py:48] [16] global_step=16, grad_norm=0.371814, loss=6.90772 +I0915 10:10:56.171278 139691287286976 submission.py:307] 16) loss = 6.908, grad_norm = 0.372 +I0915 10:10:56.439300 139636861036288 logging_writer.py:48] [17] global_step=17, grad_norm=0.362865, loss=6.90771 +I0915 10:10:56.445005 139691287286976 submission.py:307] 17) loss = 6.908, grad_norm = 0.363 +I0915 10:10:56.678069 139636869428992 logging_writer.py:48] [18] global_step=18, grad_norm=0.378814, loss=6.90773 +I0915 10:10:56.682251 139691287286976 submission.py:307] 18) loss = 6.908, grad_norm = 0.379 +I0915 10:10:56.926976 139636861036288 logging_writer.py:48] [19] global_step=19, grad_norm=0.358103, loss=6.90773 +I0915 10:10:56.931608 139691287286976 submission.py:307] 19) loss = 6.908, grad_norm = 0.358 +I0915 10:10:57.159617 139636869428992 logging_writer.py:48] [20] global_step=20, grad_norm=0.365492, loss=6.9077 +I0915 10:10:57.164705 139691287286976 submission.py:307] 20) loss = 6.908, grad_norm = 0.365 +I0915 10:10:57.423389 139636861036288 logging_writer.py:48] [21] global_step=21, grad_norm=0.365922, loss=6.90768 +I0915 10:10:57.426636 139691287286976 submission.py:307] 21) loss = 6.908, grad_norm = 0.366 +I0915 10:10:57.668408 139636869428992 logging_writer.py:48] [22] global_step=22, grad_norm=0.378326, loss=6.90769 +I0915 10:10:57.672965 139691287286976 submission.py:307] 22) loss = 6.908, grad_norm = 0.378 +I0915 10:10:57.906202 139636861036288 logging_writer.py:48] [23] global_step=23, grad_norm=0.366921, loss=6.90765 +I0915 10:10:57.910501 139691287286976 submission.py:307] 23) loss = 6.908, grad_norm = 0.367 +I0915 10:10:58.137078 139636869428992 logging_writer.py:48] [24] global_step=24, grad_norm=0.377155, loss=6.90761 +I0915 10:10:58.140903 139691287286976 submission.py:307] 24) loss = 6.908, grad_norm = 0.377 +I0915 10:10:58.357828 139636861036288 logging_writer.py:48] [25] global_step=25, grad_norm=0.364849, loss=6.90761 +I0915 10:10:58.361873 139691287286976 submission.py:307] 25) loss = 6.908, grad_norm = 0.365 +I0915 10:10:58.620797 139636869428992 logging_writer.py:48] [26] global_step=26, grad_norm=0.381582, loss=6.90758 +I0915 10:10:58.625208 139691287286976 submission.py:307] 26) loss = 6.908, grad_norm = 0.382 +I0915 10:10:58.859295 139636861036288 logging_writer.py:48] [27] global_step=27, grad_norm=0.380305, loss=6.90762 +I0915 10:10:58.865814 139691287286976 submission.py:307] 27) loss = 6.908, grad_norm = 0.380 +I0915 10:10:59.104299 139636869428992 logging_writer.py:48] [28] global_step=28, grad_norm=0.378252, loss=6.90762 +I0915 10:10:59.107610 139691287286976 submission.py:307] 28) loss = 6.908, grad_norm = 0.378 +I0915 10:10:59.334153 139636861036288 logging_writer.py:48] [29] global_step=29, grad_norm=0.362416, loss=6.90768 +I0915 10:10:59.337617 139691287286976 submission.py:307] 29) loss = 6.908, grad_norm = 0.362 +I0915 10:10:59.567507 139636869428992 logging_writer.py:48] [30] global_step=30, grad_norm=0.38648, loss=6.9075 +I0915 10:10:59.571588 139691287286976 submission.py:307] 30) loss = 6.908, grad_norm = 0.386 +I0915 10:10:59.807234 139636861036288 logging_writer.py:48] [31] global_step=31, grad_norm=0.354129, loss=6.90766 +I0915 10:10:59.811094 139691287286976 submission.py:307] 31) loss = 6.908, grad_norm = 0.354 +I0915 10:11:00.054845 139636869428992 logging_writer.py:48] [32] global_step=32, grad_norm=0.373492, loss=6.90758 +I0915 10:11:00.058091 139691287286976 submission.py:307] 32) loss = 6.908, grad_norm = 0.373 +I0915 10:11:00.298706 139636861036288 logging_writer.py:48] [33] global_step=33, grad_norm=0.367698, loss=6.90751 +I0915 10:11:00.302046 139691287286976 submission.py:307] 33) loss = 6.908, grad_norm = 0.368 +I0915 10:11:00.537246 139636869428992 logging_writer.py:48] [34] global_step=34, grad_norm=0.371406, loss=6.90744 +I0915 10:11:00.541452 139691287286976 submission.py:307] 34) loss = 6.907, grad_norm = 0.371 +I0915 10:11:00.776149 139636861036288 logging_writer.py:48] [35] global_step=35, grad_norm=0.362586, loss=6.90749 +I0915 10:11:00.780712 139691287286976 submission.py:307] 35) loss = 6.907, grad_norm = 0.363 +I0915 10:11:01.005133 139636869428992 logging_writer.py:48] [36] global_step=36, grad_norm=0.388578, loss=6.9074 +I0915 10:11:01.009266 139691287286976 submission.py:307] 36) loss = 6.907, grad_norm = 0.389 +I0915 10:11:01.231917 139636861036288 logging_writer.py:48] [37] global_step=37, grad_norm=0.373371, loss=6.90743 +I0915 10:11:01.235490 139691287286976 submission.py:307] 37) loss = 6.907, grad_norm = 0.373 +I0915 10:11:01.456712 139636869428992 logging_writer.py:48] [38] global_step=38, grad_norm=0.365087, loss=6.9074 +I0915 10:11:01.462503 139691287286976 submission.py:307] 38) loss = 6.907, grad_norm = 0.365 +I0915 10:11:01.720508 139636861036288 logging_writer.py:48] [39] global_step=39, grad_norm=0.353478, loss=6.90749 +I0915 10:11:01.724551 139691287286976 submission.py:307] 39) loss = 6.907, grad_norm = 0.353 +I0915 10:11:01.967991 139636869428992 logging_writer.py:48] [40] global_step=40, grad_norm=0.397177, loss=6.90731 +I0915 10:11:01.972762 139691287286976 submission.py:307] 40) loss = 6.907, grad_norm = 0.397 +I0915 10:11:02.216075 139636861036288 logging_writer.py:48] [41] global_step=41, grad_norm=0.3675, loss=6.90731 +I0915 10:11:02.219244 139691287286976 submission.py:307] 41) loss = 6.907, grad_norm = 0.367 +I0915 10:11:02.447047 139636869428992 logging_writer.py:48] [42] global_step=42, grad_norm=0.386728, loss=6.90726 +I0915 10:11:02.451175 139691287286976 submission.py:307] 42) loss = 6.907, grad_norm = 0.387 +I0915 10:11:02.697481 139636861036288 logging_writer.py:48] [43] global_step=43, grad_norm=0.39392, loss=6.9072 +I0915 10:11:02.703216 139691287286976 submission.py:307] 43) loss = 6.907, grad_norm = 0.394 +I0915 10:11:02.934882 139636869428992 logging_writer.py:48] [44] global_step=44, grad_norm=0.388716, loss=6.90702 +I0915 10:11:02.938201 139691287286976 submission.py:307] 44) loss = 6.907, grad_norm = 0.389 +I0915 10:11:03.169985 139636861036288 logging_writer.py:48] [45] global_step=45, grad_norm=0.342717, loss=6.90734 +I0915 10:11:03.174925 139691287286976 submission.py:307] 45) loss = 6.907, grad_norm = 0.343 +I0915 10:11:03.404135 139636869428992 logging_writer.py:48] [46] global_step=46, grad_norm=0.396949, loss=6.90697 +I0915 10:11:03.407431 139691287286976 submission.py:307] 46) loss = 6.907, grad_norm = 0.397 +I0915 10:11:03.639158 139636861036288 logging_writer.py:48] [47] global_step=47, grad_norm=0.405188, loss=6.90681 +I0915 10:11:03.644064 139691287286976 submission.py:307] 47) loss = 6.907, grad_norm = 0.405 +I0915 10:11:03.876662 139636869428992 logging_writer.py:48] [48] global_step=48, grad_norm=0.367676, loss=6.90704 +I0915 10:11:03.881566 139691287286976 submission.py:307] 48) loss = 6.907, grad_norm = 0.368 +I0915 10:11:04.124765 139636861036288 logging_writer.py:48] [49] global_step=49, grad_norm=0.365599, loss=6.90705 +I0915 10:11:04.128780 139691287286976 submission.py:307] 49) loss = 6.907, grad_norm = 0.366 +I0915 10:11:04.378962 139636869428992 logging_writer.py:48] [50] global_step=50, grad_norm=0.402577, loss=6.90669 +I0915 10:11:04.383030 139691287286976 submission.py:307] 50) loss = 6.907, grad_norm = 0.403 +I0915 10:11:04.611482 139636861036288 logging_writer.py:48] [51] global_step=51, grad_norm=0.400657, loss=6.9067 +I0915 10:11:04.615106 139691287286976 submission.py:307] 51) loss = 6.907, grad_norm = 0.401 +I0915 10:11:04.846317 139636869428992 logging_writer.py:48] [52] global_step=52, grad_norm=0.348868, loss=6.90706 +I0915 10:11:04.849545 139691287286976 submission.py:307] 52) loss = 6.907, grad_norm = 0.349 +I0915 10:11:05.088209 139636861036288 logging_writer.py:48] [53] global_step=53, grad_norm=0.412752, loss=6.90645 +I0915 10:11:05.091908 139691287286976 submission.py:307] 53) loss = 6.906, grad_norm = 0.413 +I0915 10:11:05.316556 139636869428992 logging_writer.py:48] [54] global_step=54, grad_norm=0.400848, loss=6.90649 +I0915 10:11:05.320831 139691287286976 submission.py:307] 54) loss = 6.906, grad_norm = 0.401 +I0915 10:11:05.556851 139636861036288 logging_writer.py:48] [55] global_step=55, grad_norm=0.355345, loss=6.90691 +I0915 10:11:05.560146 139691287286976 submission.py:307] 55) loss = 6.907, grad_norm = 0.355 +I0915 10:11:05.786701 139636869428992 logging_writer.py:48] [56] global_step=56, grad_norm=0.408288, loss=6.90637 +I0915 10:11:05.790567 139691287286976 submission.py:307] 56) loss = 6.906, grad_norm = 0.408 +I0915 10:11:06.063601 139636861036288 logging_writer.py:48] [57] global_step=57, grad_norm=0.422104, loss=6.90623 +I0915 10:11:06.068234 139691287286976 submission.py:307] 57) loss = 6.906, grad_norm = 0.422 +I0915 10:11:06.310164 139636869428992 logging_writer.py:48] [58] global_step=58, grad_norm=0.41864, loss=6.90615 +I0915 10:11:06.314331 139691287286976 submission.py:307] 58) loss = 6.906, grad_norm = 0.419 +I0915 10:11:06.545895 139636861036288 logging_writer.py:48] [59] global_step=59, grad_norm=0.42563, loss=6.90594 +I0915 10:11:06.549475 139691287286976 submission.py:307] 59) loss = 6.906, grad_norm = 0.426 +I0915 10:11:06.782954 139636869428992 logging_writer.py:48] [60] global_step=60, grad_norm=0.41163, loss=6.90599 +I0915 10:11:06.786541 139691287286976 submission.py:307] 60) loss = 6.906, grad_norm = 0.412 +I0915 10:11:07.018548 139636861036288 logging_writer.py:48] [61] global_step=61, grad_norm=0.429331, loss=6.90546 +I0915 10:11:07.022010 139691287286976 submission.py:307] 61) loss = 6.905, grad_norm = 0.429 +I0915 10:11:07.253911 139636869428992 logging_writer.py:48] [62] global_step=62, grad_norm=0.39019, loss=6.90603 +I0915 10:11:07.258144 139691287286976 submission.py:307] 62) loss = 6.906, grad_norm = 0.390 +I0915 10:11:07.486762 139636861036288 logging_writer.py:48] [63] global_step=63, grad_norm=0.385807, loss=6.90562 +I0915 10:11:07.490262 139691287286976 submission.py:307] 63) loss = 6.906, grad_norm = 0.386 +I0915 10:11:07.728010 139636869428992 logging_writer.py:48] [64] global_step=64, grad_norm=0.415746, loss=6.90516 +I0915 10:11:07.732982 139691287286976 submission.py:307] 64) loss = 6.905, grad_norm = 0.416 +I0915 10:11:07.968692 139636861036288 logging_writer.py:48] [65] global_step=65, grad_norm=0.396844, loss=6.90581 +I0915 10:11:07.972476 139691287286976 submission.py:307] 65) loss = 6.906, grad_norm = 0.397 +I0915 10:11:08.212708 139636869428992 logging_writer.py:48] [66] global_step=66, grad_norm=0.436445, loss=6.90461 +I0915 10:11:08.221841 139691287286976 submission.py:307] 66) loss = 6.905, grad_norm = 0.436 +I0915 10:11:08.455787 139636861036288 logging_writer.py:48] [67] global_step=67, grad_norm=0.410208, loss=6.90474 +I0915 10:11:08.461363 139691287286976 submission.py:307] 67) loss = 6.905, grad_norm = 0.410 +I0915 10:11:08.696157 139636869428992 logging_writer.py:48] [68] global_step=68, grad_norm=0.443854, loss=6.90439 +I0915 10:11:08.700528 139691287286976 submission.py:307] 68) loss = 6.904, grad_norm = 0.444 +I0915 10:11:08.927692 139636861036288 logging_writer.py:48] [69] global_step=69, grad_norm=0.446615, loss=6.90464 +I0915 10:11:08.932035 139691287286976 submission.py:307] 69) loss = 6.905, grad_norm = 0.447 +I0915 10:11:09.167318 139636869428992 logging_writer.py:48] [70] global_step=70, grad_norm=0.387827, loss=6.90487 +I0915 10:11:09.177654 139691287286976 submission.py:307] 70) loss = 6.905, grad_norm = 0.388 +I0915 10:11:09.404481 139636861036288 logging_writer.py:48] [71] global_step=71, grad_norm=0.462122, loss=6.90431 +I0915 10:11:09.407954 139691287286976 submission.py:307] 71) loss = 6.904, grad_norm = 0.462 +I0915 10:11:09.625653 139636869428992 logging_writer.py:48] [72] global_step=72, grad_norm=0.466788, loss=6.90334 +I0915 10:11:09.628872 139691287286976 submission.py:307] 72) loss = 6.903, grad_norm = 0.467 +I0915 10:11:09.892927 139636861036288 logging_writer.py:48] [73] global_step=73, grad_norm=0.464832, loss=6.90385 +I0915 10:11:09.897360 139691287286976 submission.py:307] 73) loss = 6.904, grad_norm = 0.465 +I0915 10:11:10.128032 139636869428992 logging_writer.py:48] [74] global_step=74, grad_norm=0.462631, loss=6.90372 +I0915 10:11:10.132266 139691287286976 submission.py:307] 74) loss = 6.904, grad_norm = 0.463 +I0915 10:11:10.359502 139636861036288 logging_writer.py:48] [75] global_step=75, grad_norm=0.392963, loss=6.90435 +I0915 10:11:10.364439 139691287286976 submission.py:307] 75) loss = 6.904, grad_norm = 0.393 +I0915 10:11:10.598786 139636869428992 logging_writer.py:48] [76] global_step=76, grad_norm=0.397234, loss=6.90422 +I0915 10:11:10.603067 139691287286976 submission.py:307] 76) loss = 6.904, grad_norm = 0.397 +I0915 10:11:10.832473 139636861036288 logging_writer.py:48] [77] global_step=77, grad_norm=0.408307, loss=6.9036 +I0915 10:11:10.837189 139691287286976 submission.py:307] 77) loss = 6.904, grad_norm = 0.408 +I0915 10:11:11.069481 139636869428992 logging_writer.py:48] [78] global_step=78, grad_norm=0.488536, loss=6.90255 +I0915 10:11:11.073813 139691287286976 submission.py:307] 78) loss = 6.903, grad_norm = 0.489 +I0915 10:11:11.338258 139636861036288 logging_writer.py:48] [79] global_step=79, grad_norm=0.481768, loss=6.90454 +I0915 10:11:11.344104 139691287286976 submission.py:307] 79) loss = 6.905, grad_norm = 0.482 +I0915 10:11:11.583511 139636869428992 logging_writer.py:48] [80] global_step=80, grad_norm=0.470298, loss=6.90317 +I0915 10:11:11.587335 139691287286976 submission.py:307] 80) loss = 6.903, grad_norm = 0.470 +I0915 10:11:14.424469 139636861036288 logging_writer.py:48] [81] global_step=81, grad_norm=0.395503, loss=6.90481 +I0915 10:11:14.427571 139691287286976 submission.py:307] 81) loss = 6.905, grad_norm = 0.396 +I0915 10:11:15.132951 139636869428992 logging_writer.py:48] [82] global_step=82, grad_norm=0.463576, loss=6.90225 +I0915 10:11:15.136680 139691287286976 submission.py:307] 82) loss = 6.902, grad_norm = 0.464 +I0915 10:11:16.257138 139636861036288 logging_writer.py:48] [83] global_step=83, grad_norm=0.447937, loss=6.90294 +I0915 10:11:16.262010 139691287286976 submission.py:307] 83) loss = 6.903, grad_norm = 0.448 +I0915 10:11:17.118424 139636869428992 logging_writer.py:48] [84] global_step=84, grad_norm=0.40616, loss=6.90327 +I0915 10:11:17.125005 139691287286976 submission.py:307] 84) loss = 6.903, grad_norm = 0.406 +I0915 10:11:17.558773 139636861036288 logging_writer.py:48] [85] global_step=85, grad_norm=0.405441, loss=6.90382 +I0915 10:11:17.562744 139691287286976 submission.py:307] 85) loss = 6.904, grad_norm = 0.405 +I0915 10:11:19.511405 139636869428992 logging_writer.py:48] [86] global_step=86, grad_norm=0.499999, loss=6.90129 +I0915 10:11:19.517011 139691287286976 submission.py:307] 86) loss = 6.901, grad_norm = 0.500 +I0915 10:11:19.761618 139636861036288 logging_writer.py:48] [87] global_step=87, grad_norm=0.477676, loss=6.90119 +I0915 10:11:19.767492 139691287286976 submission.py:307] 87) loss = 6.901, grad_norm = 0.478 +I0915 10:11:19.988319 139636869428992 logging_writer.py:48] [88] global_step=88, grad_norm=0.462895, loss=6.9016 +I0915 10:11:19.994410 139691287286976 submission.py:307] 88) loss = 6.902, grad_norm = 0.463 +I0915 10:11:20.532126 139636861036288 logging_writer.py:48] [89] global_step=89, grad_norm=0.410352, loss=6.90217 +I0915 10:11:20.535303 139691287286976 submission.py:307] 89) loss = 6.902, grad_norm = 0.410 +I0915 10:11:20.810440 139636869428992 logging_writer.py:48] [90] global_step=90, grad_norm=0.490264, loss=6.90056 +I0915 10:11:20.815313 139691287286976 submission.py:307] 90) loss = 6.901, grad_norm = 0.490 +I0915 10:11:21.035767 139636861036288 logging_writer.py:48] [91] global_step=91, grad_norm=0.451432, loss=6.90049 +I0915 10:11:21.038952 139691287286976 submission.py:307] 91) loss = 6.900, grad_norm = 0.451 +I0915 10:11:21.605236 139636869428992 logging_writer.py:48] [92] global_step=92, grad_norm=0.493652, loss=6.89988 +I0915 10:11:21.608393 139691287286976 submission.py:307] 92) loss = 6.900, grad_norm = 0.494 +I0915 10:11:22.155008 139636861036288 logging_writer.py:48] [93] global_step=93, grad_norm=0.48032, loss=6.90036 +I0915 10:11:22.159046 139691287286976 submission.py:307] 93) loss = 6.900, grad_norm = 0.480 +I0915 10:11:22.423153 139636869428992 logging_writer.py:48] [94] global_step=94, grad_norm=0.497711, loss=6.90046 +I0915 10:11:22.426506 139691287286976 submission.py:307] 94) loss = 6.900, grad_norm = 0.498 +I0915 10:11:22.667288 139636861036288 logging_writer.py:48] [95] global_step=95, grad_norm=0.499999, loss=6.89987 +I0915 10:11:22.670613 139691287286976 submission.py:307] 95) loss = 6.900, grad_norm = 0.500 +I0915 10:11:22.930868 139636869428992 logging_writer.py:48] [96] global_step=96, grad_norm=0.499999, loss=6.89761 +I0915 10:11:22.935100 139691287286976 submission.py:307] 96) loss = 6.898, grad_norm = 0.500 +I0915 10:11:23.772529 139636861036288 logging_writer.py:48] [97] global_step=97, grad_norm=0.447004, loss=6.90155 +I0915 10:11:23.777789 139691287286976 submission.py:307] 97) loss = 6.902, grad_norm = 0.447 +I0915 10:11:24.038126 139636869428992 logging_writer.py:48] [98] global_step=98, grad_norm=0.483873, loss=6.89845 +I0915 10:11:24.043197 139691287286976 submission.py:307] 98) loss = 6.898, grad_norm = 0.484 +I0915 10:11:24.296544 139636861036288 logging_writer.py:48] [99] global_step=99, grad_norm=0.47556, loss=6.89836 +I0915 10:11:24.299815 139691287286976 submission.py:307] 99) loss = 6.898, grad_norm = 0.476 +I0915 10:11:24.547358 139636869428992 logging_writer.py:48] [100] global_step=100, grad_norm=0.49111, loss=6.89835 +I0915 10:11:24.551347 139691287286976 submission.py:307] 100) loss = 6.898, grad_norm = 0.491 +I0915 10:19:31.096694 139636861036288 logging_writer.py:48] [500] global_step=500, grad_norm=0.499999, loss=6.73335 +I0915 10:19:31.100468 139691287286976 submission.py:307] 500) loss = 6.733, grad_norm = 0.500 +I0915 10:29:42.646139 139636869428992 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.5, loss=6.41502 +I0915 10:29:42.650444 139691287286976 submission.py:307] 1000) loss = 6.415, grad_norm = 0.500 +I0915 10:37:51.073716 139636861036288 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.5, loss=6.23349 +I0915 10:37:51.077718 139691287286976 submission.py:307] 1500) loss = 6.233, grad_norm = 0.500 +I0915 10:46:06.811997 139636869428992 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.5, loss=5.98352 +I0915 10:46:06.816135 139691287286976 submission.py:307] 2000) loss = 5.984, grad_norm = 0.500 +I0915 10:53:45.373596 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 10:55:23.518404 139691287286976 spec.py:346] Evaluating on the validation split. +I0915 10:56:29.782043 139691287286976 spec.py:363] Evaluating on the test split. +I0915 10:56:31.892200 139691287286976 submission_runner.py:516] Time since start: 3129.84s, Step: 2371, {'train/accuracy': 0.055859375, 'train/loss': 5.700372314453125, 'validation/accuracy': 0.0493, 'validation/loss': 5.87810625, 'validation/num_examples': 50000, 'test/accuracy': 0.0486, 'test/loss': 5.6401859375, 'test/num_examples': 10000, 'score': 2692.0896463394165, 'total_duration': 3129.844084262848, 'accumulated_submission_time': 2692.0896463394165, 'accumulated_eval_time': 433.4192156791687, 'accumulated_logging_time': 0.23308897018432617} +I0915 10:56:31.950789 139630921901824 logging_writer.py:48] [2371] accumulated_eval_time=433.419, accumulated_logging_time=0.233089, accumulated_submission_time=2692.09, global_step=2371, preemption_count=0, score=2692.09, test/accuracy=0.0486, test/loss=5.64019, test/num_examples=10000, total_duration=3129.84, train/accuracy=0.0558594, train/loss=5.70037, validation/accuracy=0.0493, validation/loss=5.87811, validation/num_examples=50000 +I0915 10:57:34.248839 139630930294528 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.5, loss=6.51827 +I0915 10:57:34.252084 139691287286976 submission.py:307] 2500) loss = 6.518, grad_norm = 0.500 +I0915 11:04:10.102490 139630921901824 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.5, loss=5.58079 +I0915 11:04:10.106484 139691287286976 submission.py:307] 3000) loss = 5.581, grad_norm = 0.500 +I0915 11:12:47.484114 139630930294528 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.499999, loss=6.14454 +I0915 11:12:47.488231 139691287286976 submission.py:307] 3500) loss = 6.145, grad_norm = 0.500 +I0915 11:19:21.642633 139630921901824 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.5, loss=5.22803 +I0915 11:19:21.647924 139691287286976 submission.py:307] 4000) loss = 5.228, grad_norm = 0.500 +I0915 11:25:01.792281 139630930294528 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.5, loss=4.97837 +I0915 11:25:01.796690 139691287286976 submission.py:307] 4500) loss = 4.978, grad_norm = 0.500 +I0915 11:33:52.858487 139630921901824 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.5, loss=4.9876 +I0915 11:33:52.862213 139691287286976 submission.py:307] 5000) loss = 4.988, grad_norm = 0.500 +I0915 11:38:07.826397 139630930294528 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.5, loss=4.65183 +I0915 11:38:07.830463 139691287286976 submission.py:307] 5500) loss = 4.652, grad_norm = 0.500 +I0915 11:39:26.510539 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 11:40:49.685362 139691287286976 spec.py:346] Evaluating on the validation split. +I0915 11:41:55.598844 139691287286976 spec.py:363] Evaluating on the test split. +I0915 11:41:56.541747 139691287286976 submission_runner.py:516] Time since start: 5854.49s, Step: 5610, {'train/accuracy': 0.189765625, 'train/loss': 4.9383306884765625, 'validation/accuracy': 0.17356, 'validation/loss': 4.7599465625, 'validation/num_examples': 50000, 'test/accuracy': 0.1741, 'test/loss': 4.29425546875, 'test/num_examples': 10000, 'score': 5262.350856781006, 'total_duration': 5854.4937126636505, 'accumulated_submission_time': 5262.350856781006, 'accumulated_eval_time': 583.4502947330475, 'accumulated_logging_time': 0.3073396682739258} +I0915 11:41:56.648062 139670583207680 logging_writer.py:48] [5610] accumulated_eval_time=583.45, accumulated_logging_time=0.30734, accumulated_submission_time=5262.35, global_step=5610, preemption_count=0, score=5262.35, test/accuracy=0.1741, test/loss=4.29426, test/num_examples=10000, total_duration=5854.49, train/accuracy=0.189766, train/loss=4.93833, validation/accuracy=0.17356, validation/loss=4.75995, validation/num_examples=50000 +I0915 11:47:49.085018 139671086556928 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.5, loss=4.45667 +I0915 11:47:49.091848 139691287286976 submission.py:307] 6000) loss = 4.457, grad_norm = 0.500 +I0915 11:54:34.254230 139670583207680 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.5, loss=4.45879 +I0915 11:54:34.259073 139691287286976 submission.py:307] 6500) loss = 4.459, grad_norm = 0.500 +I0915 12:00:07.991342 139671086556928 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.5, loss=4.19093 +I0915 12:00:07.995583 139691287286976 submission.py:307] 7000) loss = 4.191, grad_norm = 0.500 +I0915 12:08:51.215572 139670583207680 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.5, loss=4.19564 +I0915 12:08:51.221849 139691287286976 submission.py:307] 7500) loss = 4.196, grad_norm = 0.500 +I0915 12:13:11.912562 139671086556928 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.5, loss=4.29967 +I0915 12:13:11.919200 139691287286976 submission.py:307] 8000) loss = 4.300, grad_norm = 0.500 +I0915 12:20:12.050013 139670583207680 logging_writer.py:48] [8500] global_step=8500, grad_norm=0.5, loss=5.9518 +I0915 12:20:12.053866 139691287286976 submission.py:307] 8500) loss = 5.952, grad_norm = 0.500 +I0915 12:24:52.323828 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 12:26:08.354708 139691287286976 spec.py:346] Evaluating on the validation split. +I0915 12:27:13.761843 139691287286976 spec.py:363] Evaluating on the test split. +I0915 12:27:14.730298 139691287286976 submission_runner.py:516] Time since start: 8572.68s, Step: 8777, {'train/accuracy': 0.324140625, 'train/loss': 4.036053466796875, 'validation/accuracy': 0.287, 'validation/loss': 4.09793375, 'validation/num_examples': 50000, 'test/accuracy': 0.2762, 'test/loss': 3.52323125, 'test/num_examples': 10000, 'score': 7834.110359668732, 'total_duration': 8572.682470083237, 'accumulated_submission_time': 7834.110359668732, 'accumulated_eval_time': 725.8568375110626, 'accumulated_logging_time': 0.42612242698669434} +I0915 12:27:14.866101 139671044593408 logging_writer.py:48] [8777] accumulated_eval_time=725.857, accumulated_logging_time=0.426122, accumulated_submission_time=7834.11, global_step=8777, preemption_count=0, score=7834.11, test/accuracy=0.2762, test/loss=3.52323, test/num_examples=10000, total_duration=8572.68, train/accuracy=0.324141, train/loss=4.03605, validation/accuracy=0.287, validation/loss=4.09793, validation/num_examples=50000 +I0915 12:29:36.914781 139670952273664 logging_writer.py:48] [9000] global_step=9000, grad_norm=0.5, loss=3.82983 +I0915 12:29:36.918842 139691287286976 submission.py:307] 9000) loss = 3.830, grad_norm = 0.500 +I0915 12:37:03.341185 139671044593408 logging_writer.py:48] [9500] global_step=9500, grad_norm=0.5, loss=4.93496 +I0915 12:37:03.346070 139691287286976 submission.py:307] 9500) loss = 4.935, grad_norm = 0.500 +I0915 12:46:07.764296 139670952273664 logging_writer.py:48] [10000] global_step=10000, grad_norm=0.5, loss=3.53752 +I0915 12:46:08.278967 139691287286976 submission.py:307] 10000) loss = 3.538, grad_norm = 0.500 +I0915 12:50:24.221281 139671044593408 logging_writer.py:48] [10500] global_step=10500, grad_norm=0.5, loss=3.91813 +I0915 12:50:24.448169 139691287286976 submission.py:307] 10500) loss = 3.918, grad_norm = 0.500 +I0915 12:58:09.106225 139670952273664 logging_writer.py:48] [11000] global_step=11000, grad_norm=0.5, loss=5.31797 +I0915 12:58:09.110142 139691287286976 submission.py:307] 11000) loss = 5.318, grad_norm = 0.500 +I0915 13:04:49.306410 139671044593408 logging_writer.py:48] [11500] global_step=11500, grad_norm=0.5, loss=3.14241 +I0915 13:04:49.311098 139691287286976 submission.py:307] 11500) loss = 3.142, grad_norm = 0.500 +I0915 13:10:03.071184 139670952273664 logging_writer.py:48] [12000] global_step=12000, grad_norm=0.5, loss=3.16615 +I0915 13:10:03.075547 139691287286976 submission.py:307] 12000) loss = 3.166, grad_norm = 0.500 +I0915 13:10:20.947241 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 13:11:33.649076 139691287286976 spec.py:346] Evaluating on the validation split. +I0915 13:12:39.425945 139691287286976 spec.py:363] Evaluating on the test split. +I0915 13:12:40.336635 139691287286976 submission_runner.py:516] Time since start: 11298.29s, Step: 12004, {'train/accuracy': 0.40556640625, 'train/loss': 3.7709576416015627, 'validation/accuracy': 0.36128, 'validation/loss': 3.826253125, 'validation/num_examples': 50000, 'test/accuracy': 0.3526, 'test/loss': 3.036040234375, 'test/num_examples': 10000, 'score': 10411.95853638649, 'total_duration': 11298.288755893707, 'accumulated_submission_time': 10411.95853638649, 'accumulated_eval_time': 865.246161699295, 'accumulated_logging_time': 0.5769710540771484} +I0915 13:12:40.601176 139671027808000 logging_writer.py:48] [12004] accumulated_eval_time=865.246, accumulated_logging_time=0.576971, accumulated_submission_time=10412, global_step=12004, preemption_count=0, score=10412, test/accuracy=0.3526, test/loss=3.03604, test/num_examples=10000, total_duration=11298.3, train/accuracy=0.405566, train/loss=3.77096, validation/accuracy=0.36128, validation/loss=3.82625, validation/num_examples=50000 +I0915 13:20:37.020191 139670985844480 logging_writer.py:48] [12500] global_step=12500, grad_norm=0.5, loss=3.82578 +I0915 13:20:37.025835 139691287286976 submission.py:307] 12500) loss = 3.826, grad_norm = 0.500 +I0915 13:25:30.641490 139671027808000 logging_writer.py:48] [13000] global_step=13000, grad_norm=0.5, loss=3.13331 +I0915 13:25:30.645596 139691287286976 submission.py:307] 13000) loss = 3.133, grad_norm = 0.500 +I0915 13:32:59.616824 139670985844480 logging_writer.py:48] [13500] global_step=13500, grad_norm=0.5, loss=3.37895 +I0915 13:32:59.625049 139691287286976 submission.py:307] 13500) loss = 3.379, grad_norm = 0.500 +I0915 13:39:27.625791 139671027808000 logging_writer.py:48] [14000] global_step=14000, grad_norm=0.5, loss=2.89204 +I0915 13:39:27.633557 139691287286976 submission.py:307] 14000) loss = 2.892, grad_norm = 0.500 +I0915 13:44:50.447182 139670985844480 logging_writer.py:48] [14500] global_step=14500, grad_norm=0.5, loss=2.91157 +I0915 13:44:50.451058 139691287286976 submission.py:307] 14500) loss = 2.912, grad_norm = 0.500 +I0915 13:53:22.006879 139671027808000 logging_writer.py:48] [15000] global_step=15000, grad_norm=0.5, loss=3.89145 +I0915 13:53:22.010751 139691287286976 submission.py:307] 15000) loss = 3.891, grad_norm = 0.500 +I0915 13:55:36.313232 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 13:56:52.935414 139691287286976 spec.py:346] Evaluating on the validation split. +I0915 13:57:59.113830 139691287286976 spec.py:363] Evaluating on the test split. +I0915 13:58:00.037979 139691287286976 submission_runner.py:516] Time since start: 14017.99s, Step: 15268, {'train/accuracy': 0.479921875, 'train/loss': 3.2422940063476564, 'validation/accuracy': 0.41712, 'validation/loss': 3.508525, 'validation/num_examples': 50000, 'test/accuracy': 0.4116, 'test/loss': 2.7012171875, 'test/num_examples': 10000, 'score': 12979.315444469452, 'total_duration': 14017.990199327469, 'accumulated_submission_time': 12979.315444469452, 'accumulated_eval_time': 1008.971207857132, 'accumulated_logging_time': 0.8551476001739502} +I0915 13:58:00.329049 139671933810432 logging_writer.py:48] [15268] accumulated_eval_time=1008.97, accumulated_logging_time=0.855148, accumulated_submission_time=12979.3, global_step=15268, preemption_count=0, score=12979.3, test/accuracy=0.4116, test/loss=2.70122, test/num_examples=10000, total_duration=14018, train/accuracy=0.479922, train/loss=3.24229, validation/accuracy=0.41712, validation/loss=3.50853, validation/num_examples=50000 +I0915 14:00:49.118320 139670960666368 logging_writer.py:48] [15500] global_step=15500, grad_norm=0.5, loss=2.65176 +I0915 14:00:49.125400 139691287286976 submission.py:307] 15500) loss = 2.652, grad_norm = 0.500 +I0915 14:09:26.849026 139671933810432 logging_writer.py:48] [16000] global_step=16000, grad_norm=0.5, loss=2.68837 +I0915 14:09:26.852895 139691287286976 submission.py:307] 16000) loss = 2.688, grad_norm = 0.500 +I0915 14:16:24.526199 139670960666368 logging_writer.py:48] [16500] global_step=16500, grad_norm=0.5, loss=3.24386 +I0915 14:16:24.545804 139691287286976 submission.py:307] 16500) loss = 3.244, grad_norm = 0.500 +I0915 14:21:43.870905 139671933810432 logging_writer.py:48] [17000] global_step=17000, grad_norm=0.5, loss=3.99592 +I0915 14:21:43.875227 139691287286976 submission.py:307] 17000) loss = 3.996, grad_norm = 0.500 +I0915 14:30:45.230170 139670960666368 logging_writer.py:48] [17500] global_step=17500, grad_norm=0.5, loss=2.80049 +I0915 14:30:45.235483 139691287286976 submission.py:307] 17500) loss = 2.800, grad_norm = 0.500 +I0915 14:35:14.400601 139671933810432 logging_writer.py:48] [18000] global_step=18000, grad_norm=0.5, loss=4.22366 +I0915 14:35:14.406405 139691287286976 submission.py:307] 18000) loss = 4.224, grad_norm = 0.500 +I0915 14:40:56.146348 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 14:42:26.379940 139691287286976 spec.py:346] Evaluating on the validation split. +I0915 14:43:32.281650 139691287286976 spec.py:363] Evaluating on the test split. +I0915 14:43:33.188815 139691287286976 submission_runner.py:516] Time since start: 16751.14s, Step: 18417, {'train/accuracy': 0.54462890625, 'train/loss': 2.800125732421875, 'validation/accuracy': 0.46806, 'validation/loss': 3.2255234375, 'validation/num_examples': 50000, 'test/accuracy': 0.4518, 'test/loss': 2.47650390625, 'test/num_examples': 10000, 'score': 15546.428710222244, 'total_duration': 16751.14104294777, 'accumulated_submission_time': 15546.428710222244, 'accumulated_eval_time': 1166.0138342380524, 'accumulated_logging_time': 1.1592702865600586} +I0915 14:43:33.725684 139670943880960 logging_writer.py:48] [18417] accumulated_eval_time=1166.01, accumulated_logging_time=1.15927, accumulated_submission_time=15546.4, global_step=18417, preemption_count=0, score=15546.4, test/accuracy=0.4518, test/loss=2.4765, test/num_examples=10000, total_duration=16751.1, train/accuracy=0.544629, train/loss=2.80013, validation/accuracy=0.46806, validation/loss=3.22552, validation/num_examples=50000 +I0915 14:43:57.572397 139671027808000 logging_writer.py:48] [18500] global_step=18500, grad_norm=0.5, loss=2.86881 +I0915 14:43:57.577773 139691287286976 submission.py:307] 18500) loss = 2.869, grad_norm = 0.500 +I0915 14:51:18.535990 139670943880960 logging_writer.py:48] [19000] global_step=19000, grad_norm=0.5, loss=5.03151 +I0915 14:51:18.541355 139691287286976 submission.py:307] 19000) loss = 5.032, grad_norm = 0.500 +I0915 14:57:11.733577 139671027808000 logging_writer.py:48] [19500] global_step=19500, grad_norm=0.499999, loss=4.90259 +I0915 14:57:11.737531 139691287286976 submission.py:307] 19500) loss = 4.903, grad_norm = 0.500 +I0915 15:05:44.798869 139670943880960 logging_writer.py:48] [20000] global_step=20000, grad_norm=0.5, loss=2.42998 +I0915 15:05:44.802844 139691287286976 submission.py:307] 20000) loss = 2.430, grad_norm = 0.500 +I0915 15:10:08.871207 139671027808000 logging_writer.py:48] [20500] global_step=20500, grad_norm=0.5, loss=2.52971 +I0915 15:10:08.876605 139691287286976 submission.py:307] 20500) loss = 2.530, grad_norm = 0.500 +I0915 15:17:30.004780 139670943880960 logging_writer.py:48] [21000] global_step=21000, grad_norm=0.5, loss=2.42083 +I0915 15:17:30.008783 139691287286976 submission.py:307] 21000) loss = 2.421, grad_norm = 0.500 +I0915 15:23:56.325068 139671027808000 logging_writer.py:48] [21500] global_step=21500, grad_norm=0.499999, loss=4.718 +I0915 15:23:56.330977 139691287286976 submission.py:307] 21500) loss = 4.718, grad_norm = 0.500 +I0915 15:26:30.278908 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 15:27:50.347056 139691287286976 spec.py:346] Evaluating on the validation split. +I0915 15:28:56.177633 139691287286976 spec.py:363] Evaluating on the test split. +I0915 15:28:57.118380 139691287286976 submission_runner.py:516] Time since start: 19475.07s, Step: 21768, {'train/accuracy': 0.54513671875, 'train/loss': 3.107976379394531, 'validation/accuracy': 0.49176, 'validation/loss': 3.171916875, 'validation/num_examples': 50000, 'test/accuracy': 0.4811, 'test/loss': 2.323434375, 'test/num_examples': 10000, 'score': 18114.06486916542, 'total_duration': 19475.07040452957, 'accumulated_submission_time': 18114.06486916542, 'accumulated_eval_time': 1312.8532564640045, 'accumulated_logging_time': 2.128077268600464} +I0915 15:28:57.392161 139671086556928 logging_writer.py:48] [21768] accumulated_eval_time=1312.85, accumulated_logging_time=2.12808, accumulated_submission_time=18114.1, global_step=21768, preemption_count=0, score=18114.1, test/accuracy=0.4811, test/loss=2.32343, test/num_examples=10000, total_duration=19475.1, train/accuracy=0.545137, train/loss=3.10798, validation/accuracy=0.49176, validation/loss=3.17192, validation/num_examples=50000 +I0915 15:31:53.773188 139671069771520 logging_writer.py:48] [22000] global_step=22000, grad_norm=0.499999, loss=5.04309 +I0915 15:31:53.778100 139691287286976 submission.py:307] 22000) loss = 5.043, grad_norm = 0.500 +I0915 15:41:02.335211 139671086556928 logging_writer.py:48] [22500] global_step=22500, grad_norm=0.5, loss=2.34218 +I0915 15:41:02.339263 139691287286976 submission.py:307] 22500) loss = 2.342, grad_norm = 0.500 +I0915 15:45:34.944192 139671069771520 logging_writer.py:48] [23000] global_step=23000, grad_norm=0.5, loss=3.8108 +I0915 15:45:34.948409 139691287286976 submission.py:307] 23000) loss = 3.811, grad_norm = 0.500 +I0915 15:52:59.512782 139671086556928 logging_writer.py:48] [23500] global_step=23500, grad_norm=0.5, loss=2.23757 +I0915 15:52:59.517780 139691287286976 submission.py:307] 23500) loss = 2.238, grad_norm = 0.500 +I0915 15:59:41.999502 139671069771520 logging_writer.py:48] [24000] global_step=24000, grad_norm=0.5, loss=4.44578 +I0915 15:59:42.004189 139691287286976 submission.py:307] 24000) loss = 4.446, grad_norm = 0.500 +I0915 16:05:01.050482 139671086556928 logging_writer.py:48] [24500] global_step=24500, grad_norm=0.5, loss=2.23962 +I0915 16:05:01.054578 139691287286976 submission.py:307] 24500) loss = 2.240, grad_norm = 0.500 +I0915 16:11:52.997965 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 16:13:25.409240 139691287286976 spec.py:346] Evaluating on the validation split. +I0915 16:14:30.914354 139691287286976 spec.py:363] Evaluating on the test split. +I0915 16:14:31.824035 139691287286976 submission_runner.py:516] Time since start: 22209.78s, Step: 24895, {'train/accuracy': 0.56482421875, 'train/loss': 3.1704031372070314, 'validation/accuracy': 0.5066, 'validation/loss': 3.2526559375, 'validation/num_examples': 50000, 'test/accuracy': 0.502, 'test/loss': 2.215092578125, 'test/num_examples': 10000, 'score': 20681.5164604187, 'total_duration': 22209.776253461838, 'accumulated_submission_time': 20681.5164604187, 'accumulated_eval_time': 1471.6794302463531, 'accumulated_logging_time': 2.4110121726989746} +I0915 16:14:32.297312 139671061378816 logging_writer.py:48] [24895] accumulated_eval_time=1471.68, accumulated_logging_time=2.41101, accumulated_submission_time=20681.5, global_step=24895, preemption_count=0, score=20681.5, test/accuracy=0.502, test/loss=2.21509, test/num_examples=10000, total_duration=22209.8, train/accuracy=0.564824, train/loss=3.1704, validation/accuracy=0.5066, validation/loss=3.25266, validation/num_examples=50000 +I0915 16:15:21.922931 139670969059072 logging_writer.py:48] [25000] global_step=25000, grad_norm=0.5, loss=3.16885 +I0915 16:15:21.927366 139691287286976 submission.py:307] 25000) loss = 3.169, grad_norm = 0.500 +I0915 16:21:38.714220 139671061378816 logging_writer.py:48] [25500] global_step=25500, grad_norm=0.499999, loss=3.69297 +I0915 16:21:38.718883 139691287286976 submission.py:307] 25500) loss = 3.693, grad_norm = 0.500 +I0915 16:29:27.411288 139670969059072 logging_writer.py:48] [26000] global_step=26000, grad_norm=0.5, loss=2.14704 +I0915 16:29:27.416929 139691287286976 submission.py:307] 26000) loss = 2.147, grad_norm = 0.500 +I0915 16:35:58.626034 139671061378816 logging_writer.py:48] [26500] global_step=26500, grad_norm=0.5, loss=2.22875 +I0915 16:35:58.630178 139691287286976 submission.py:307] 26500) loss = 2.229, grad_norm = 0.500 +I0915 16:41:26.042199 139670969059072 logging_writer.py:48] [27000] global_step=27000, grad_norm=0.5, loss=2.0943 +I0915 16:41:26.047396 139691287286976 submission.py:307] 27000) loss = 2.094, grad_norm = 0.500 +I0915 16:50:23.463670 139671061378816 logging_writer.py:48] [27500] global_step=27500, grad_norm=0.5, loss=2.36753 +I0915 16:50:23.468730 139691287286976 submission.py:307] 27500) loss = 2.368, grad_norm = 0.500 +I0915 16:54:40.580807 139670969059072 logging_writer.py:48] [28000] global_step=28000, grad_norm=0.5, loss=2.39649 +I0915 16:54:40.584845 139691287286976 submission.py:307] 28000) loss = 2.396, grad_norm = 0.500 +I0915 16:57:28.206183 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 16:58:46.999575 139691287286976 spec.py:346] Evaluating on the validation split. +I0915 16:59:52.820522 139691287286976 spec.py:363] Evaluating on the test split. +I0915 16:59:53.742830 139691287286976 submission_runner.py:516] Time since start: 24931.70s, Step: 28226, {'train/accuracy': 0.5911328125, 'train/loss': 2.883991394042969, 'validation/accuracy': 0.5363, 'validation/loss': 2.76262, 'validation/num_examples': 50000, 'test/accuracy': 0.5237, 'test/loss': 2.1259302734375, 'test/num_examples': 10000, 'score': 23248.10187292099, 'total_duration': 24931.695034742355, 'accumulated_submission_time': 23248.10187292099, 'accumulated_eval_time': 1617.216381072998, 'accumulated_logging_time': 3.4876534938812256} +I0915 16:59:54.272203 139670994237184 logging_writer.py:48] [28226] accumulated_eval_time=1617.22, accumulated_logging_time=3.48765, accumulated_submission_time=23248.1, global_step=28226, preemption_count=0, score=23248.1, test/accuracy=0.5237, test/loss=2.12593, test/num_examples=10000, total_duration=24931.7, train/accuracy=0.591133, train/loss=2.88399, validation/accuracy=0.5363, validation/loss=2.76262, validation/num_examples=50000 +I0915 17:03:41.194243 139671069771520 logging_writer.py:48] [28500] global_step=28500, grad_norm=0.5, loss=3.46755 +I0915 17:03:41.198414 139691287286976 submission.py:307] 28500) loss = 3.468, grad_norm = 0.500 +I0915 17:10:43.868266 139670994237184 logging_writer.py:48] [29000] global_step=29000, grad_norm=0.5, loss=3.9469 +I0915 17:10:43.874256 139691287286976 submission.py:307] 29000) loss = 3.947, grad_norm = 0.500 +I0915 17:16:22.639120 139671069771520 logging_writer.py:48] [29500] global_step=29500, grad_norm=0.5, loss=2.25698 +I0915 17:16:22.643159 139691287286976 submission.py:307] 29500) loss = 2.257, grad_norm = 0.500 +I0915 17:25:12.275688 139670994237184 logging_writer.py:48] [30000] global_step=30000, grad_norm=0.5, loss=3.45464 +I0915 17:25:12.280821 139691287286976 submission.py:307] 30000) loss = 3.455, grad_norm = 0.500 +I0915 17:29:28.616524 139671069771520 logging_writer.py:48] [30500] global_step=30500, grad_norm=0.499999, loss=4.51562 +I0915 17:29:28.627384 139691287286976 submission.py:307] 30500) loss = 4.516, grad_norm = 0.500 +I0915 17:36:34.779054 139670994237184 logging_writer.py:48] [31000] global_step=31000, grad_norm=0.5, loss=2.37183 +I0915 17:36:34.783256 139691287286976 submission.py:307] 31000) loss = 2.372, grad_norm = 0.500 +I0915 17:42:50.803843 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 17:44:03.441494 139691287286976 spec.py:346] Evaluating on the validation split. +I0915 17:45:09.233392 139691287286976 spec.py:363] Evaluating on the test split. +I0915 17:45:10.144285 139691287286976 submission_runner.py:516] Time since start: 27648.10s, Step: 31455, {'train/accuracy': 0.663359375, 'train/loss': 2.111064910888672, 'validation/accuracy': 0.5554, 'validation/loss': 2.83841125, 'validation/num_examples': 50000, 'test/accuracy': 0.5334, 'test/loss': 2.0450654296875, 'test/num_examples': 10000, 'score': 25816.279580831528, 'total_duration': 27648.096289157867, 'accumulated_submission_time': 25816.279580831528, 'accumulated_eval_time': 1756.5567178726196, 'accumulated_logging_time': 4.6141135692596436} +I0915 17:45:10.444992 139671086556928 logging_writer.py:48] [31455] accumulated_eval_time=1756.56, accumulated_logging_time=4.61411, accumulated_submission_time=25816.3, global_step=31455, preemption_count=0, score=25816.3, test/accuracy=0.5334, test/loss=2.04507, test/num_examples=10000, total_duration=27648.1, train/accuracy=0.663359, train/loss=2.11106, validation/accuracy=0.5554, validation/loss=2.83841, validation/num_examples=50000 +I0915 17:45:22.200914 139670977451776 logging_writer.py:48] [31500] global_step=31500, grad_norm=0.5, loss=4.4791 +I0915 17:45:22.210267 139691287286976 submission.py:307] 31500) loss = 4.479, grad_norm = 0.500 +I0915 17:53:30.562304 139671086556928 logging_writer.py:48] [32000] global_step=32000, grad_norm=0.5, loss=4.62178 +I0915 17:53:30.569060 139691287286976 submission.py:307] 32000) loss = 4.622, grad_norm = 0.500 +I0915 18:03:14.439668 139670977451776 logging_writer.py:48] [32500] global_step=32500, grad_norm=0.5, loss=2.12143 +I0915 18:03:14.443482 139691287286976 submission.py:307] 32500) loss = 2.121, grad_norm = 0.500 +I0915 18:07:58.146758 139671086556928 logging_writer.py:48] [33000] global_step=33000, grad_norm=0.5, loss=4.30037 +I0915 18:07:58.161393 139691287286976 submission.py:307] 33000) loss = 4.300, grad_norm = 0.500 +I0915 18:16:02.523919 139670977451776 logging_writer.py:48] [33500] global_step=33500, grad_norm=0.5, loss=2.13373 +I0915 18:16:02.529477 139691287286976 submission.py:307] 33500) loss = 2.134, grad_norm = 0.500 +I0915 18:23:20.921843 139671086556928 logging_writer.py:48] [34000] global_step=34000, grad_norm=0.5, loss=1.793 +I0915 18:23:20.926731 139691287286976 submission.py:307] 34000) loss = 1.793, grad_norm = 0.500 +I0915 18:28:07.042795 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 18:29:31.721522 139691287286976 spec.py:346] Evaluating on the validation split. +I0915 18:30:37.823447 139691287286976 spec.py:363] Evaluating on the test split. +I0915 18:30:38.809768 139691287286976 submission_runner.py:516] Time since start: 30376.76s, Step: 34457, {'train/accuracy': 0.625390625, 'train/loss': 2.8165191650390624, 'validation/accuracy': 0.5694, 'validation/loss': 2.614239375, 'validation/num_examples': 50000, 'test/accuracy': 0.5457, 'test/loss': 2.0043236328125, 'test/num_examples': 10000, 'score': 28384.168689012527, 'total_duration': 30376.761737585068, 'accumulated_submission_time': 28384.168689012527, 'accumulated_eval_time': 1908.323546409607, 'accumulated_logging_time': 4.924053430557251} +I0915 18:30:39.114791 139671061378816 logging_writer.py:48] [34457] accumulated_eval_time=1908.32, accumulated_logging_time=4.92405, accumulated_submission_time=28384.2, global_step=34457, preemption_count=0, score=28384.2, test/accuracy=0.5457, test/loss=2.00432, test/num_examples=10000, total_duration=30376.8, train/accuracy=0.625391, train/loss=2.81652, validation/accuracy=0.5694, validation/loss=2.61424, validation/num_examples=50000 +I0915 18:30:50.921443 139670985844480 logging_writer.py:48] [34500] global_step=34500, grad_norm=0.5, loss=2.01293 +I0915 18:30:50.924866 139691287286976 submission.py:307] 34500) loss = 2.013, grad_norm = 0.500 +I0915 18:39:39.535209 139671061378816 logging_writer.py:48] [35000] global_step=35000, grad_norm=0.5, loss=2.04595 +I0915 18:39:39.539450 139691287286976 submission.py:307] 35000) loss = 2.046, grad_norm = 0.500 +I0915 18:44:24.029840 139670985844480 logging_writer.py:48] [35500] global_step=35500, grad_norm=0.499999, loss=2.01742 +I0915 18:44:24.039645 139691287286976 submission.py:307] 35500) loss = 2.017, grad_norm = 0.500 +I0915 18:51:47.008340 139671061378816 logging_writer.py:48] [36000] global_step=36000, grad_norm=0.5, loss=3.21395 +I0915 18:51:47.013662 139691287286976 submission.py:307] 36000) loss = 3.214, grad_norm = 0.500 +I0915 18:58:30.586603 139670985844480 logging_writer.py:48] [36500] global_step=36500, grad_norm=0.499999, loss=2.3543 +I0915 18:58:30.590941 139691287286976 submission.py:307] 36500) loss = 2.354, grad_norm = 0.500 +I0915 19:03:46.500214 139671061378816 logging_writer.py:48] [37000] global_step=37000, grad_norm=0.5, loss=2.36029 +I0915 19:03:46.512952 139691287286976 submission.py:307] 37000) loss = 2.360, grad_norm = 0.500 +I0915 19:12:23.498025 139670985844480 logging_writer.py:48] [37500] global_step=37500, grad_norm=0.5, loss=1.91785 +I0915 19:12:23.503496 139691287286976 submission.py:307] 37500) loss = 1.918, grad_norm = 0.500 +I0915 19:13:34.903562 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 19:14:50.731727 139691287286976 spec.py:346] Evaluating on the validation split. +I0915 19:15:56.774115 139691287286976 spec.py:363] Evaluating on the test split. +I0915 19:15:57.763630 139691287286976 submission_runner.py:516] Time since start: 33095.72s, Step: 37631, {'train/accuracy': 0.65158203125, 'train/loss': 2.6507022094726564, 'validation/accuracy': 0.57466, 'validation/loss': 2.8467275, 'validation/num_examples': 50000, 'test/accuracy': 0.5491, 'test/loss': 1.96443515625, 'test/num_examples': 10000, 'score': 30951.68151497841, 'total_duration': 33095.715420007706, 'accumulated_submission_time': 30951.68151497841, 'accumulated_eval_time': 2051.183312892914, 'accumulated_logging_time': 5.2484130859375} +I0915 19:15:58.120376 139671094949632 logging_writer.py:48] [37631] accumulated_eval_time=2051.18, accumulated_logging_time=5.24841, accumulated_submission_time=30951.7, global_step=37631, preemption_count=0, score=30951.7, test/accuracy=0.5491, test/loss=1.96444, test/num_examples=10000, total_duration=33095.7, train/accuracy=0.651582, train/loss=2.6507, validation/accuracy=0.57466, validation/loss=2.84673, validation/num_examples=50000 +I0915 19:20:53.476018 139670943880960 logging_writer.py:48] [38000] global_step=38000, grad_norm=0.5, loss=1.81286 +I0915 19:20:53.481690 139691287286976 submission.py:307] 38000) loss = 1.813, grad_norm = 0.500 +I0915 19:29:26.034503 139671094949632 logging_writer.py:48] [38500] global_step=38500, grad_norm=0.5, loss=1.87005 +I0915 19:29:26.038473 139691287286976 submission.py:307] 38500) loss = 1.870, grad_norm = 0.500 +I0915 19:36:17.148438 139670943880960 logging_writer.py:48] [39000] global_step=39000, grad_norm=0.5, loss=1.82001 +I0915 19:36:17.160721 139691287286976 submission.py:307] 39000) loss = 1.820, grad_norm = 0.500 +I0915 19:41:31.909519 139671094949632 logging_writer.py:48] [39500] global_step=39500, grad_norm=0.5, loss=1.96536 +I0915 19:41:31.913450 139691287286976 submission.py:307] 39500) loss = 1.965, grad_norm = 0.500 +I0915 19:50:51.049668 139670943880960 logging_writer.py:48] [40000] global_step=40000, grad_norm=0.5, loss=3.18047 +I0915 19:50:51.057209 139691287286976 submission.py:307] 40000) loss = 3.180, grad_norm = 0.500 +I0915 19:55:22.960525 139671094949632 logging_writer.py:48] [40500] global_step=40500, grad_norm=0.5, loss=4.40528 +I0915 19:55:22.964886 139691287286976 submission.py:307] 40500) loss = 4.405, grad_norm = 0.500 +I0915 19:58:58.874383 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 20:00:20.242134 139691287286976 spec.py:346] Evaluating on the validation split. +I0915 20:01:26.068674 139691287286976 spec.py:363] Evaluating on the test split. +I0915 20:01:26.993849 139691287286976 submission_runner.py:516] Time since start: 35824.95s, Step: 40793, {'train/accuracy': 0.66376953125, 'train/loss': 2.469996032714844, 'validation/accuracy': 0.56806, 'validation/loss': 2.9299578125, 'validation/num_examples': 50000, 'test/accuracy': 0.5644, 'test/loss': 1.912332421875, 'test/num_examples': 10000, 'score': 33523.692170619965, 'total_duration': 35824.94554591179, 'accumulated_submission_time': 33523.692170619965, 'accumulated_eval_time': 2199.3028275966644, 'accumulated_logging_time': 5.618121862411499} +I0915 20:01:27.306797 139670960666368 logging_writer.py:48] [40793] accumulated_eval_time=2199.3, accumulated_logging_time=5.61812, accumulated_submission_time=33523.7, global_step=40793, preemption_count=0, score=33523.7, test/accuracy=0.5644, test/loss=1.91233, test/num_examples=10000, total_duration=35824.9, train/accuracy=0.66377, train/loss=2.47, validation/accuracy=0.56806, validation/loss=2.92996, validation/num_examples=50000 +I0915 20:04:08.394834 139671002629888 logging_writer.py:48] [41000] global_step=41000, grad_norm=0.5, loss=1.93251 +I0915 20:04:08.400298 139691287286976 submission.py:307] 41000) loss = 1.933, grad_norm = 0.500 +I0915 20:11:16.130718 139670960666368 logging_writer.py:48] [41500] global_step=41500, grad_norm=0.5, loss=3.18761 +I0915 20:11:16.135981 139691287286976 submission.py:307] 41500) loss = 3.188, grad_norm = 0.500 +I0915 20:16:54.021459 139671002629888 logging_writer.py:48] [42000] global_step=42000, grad_norm=0.5, loss=3.6043 +I0915 20:16:54.026351 139691287286976 submission.py:307] 42000) loss = 3.604, grad_norm = 0.500 +I0915 20:25:40.529762 139670960666368 logging_writer.py:48] [42500] global_step=42500, grad_norm=0.5, loss=1.71735 +I0915 20:25:40.538944 139691287286976 submission.py:307] 42500) loss = 1.717, grad_norm = 0.500 +I0915 20:30:08.677961 139671002629888 logging_writer.py:48] [43000] global_step=43000, grad_norm=0.5, loss=1.85781 +I0915 20:30:08.698942 139691287286976 submission.py:307] 43000) loss = 1.858, grad_norm = 0.500 +I0915 20:37:15.762866 139670960666368 logging_writer.py:48] [43500] global_step=43500, grad_norm=0.499999, loss=3.047 +I0915 20:37:15.766803 139691287286976 submission.py:307] 43500) loss = 3.047, grad_norm = 0.500 +I0915 20:43:52.677520 139671002629888 logging_writer.py:48] [44000] global_step=44000, grad_norm=0.5, loss=1.88634 +I0915 20:43:52.692064 139691287286976 submission.py:307] 44000) loss = 1.886, grad_norm = 0.500 +I0915 20:44:22.752334 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 20:45:35.690452 139691287286976 spec.py:346] Evaluating on the validation split. +I0915 20:46:41.784549 139691287286976 spec.py:363] Evaluating on the test split. +I0915 20:46:42.836011 139691287286976 submission_runner.py:516] Time since start: 38540.79s, Step: 44048, {'train/accuracy': 0.66783203125, 'train/loss': 2.4458709716796876, 'validation/accuracy': 0.5822, 'validation/loss': 2.6944565625, 'validation/num_examples': 50000, 'test/accuracy': 0.5681, 'test/loss': 1.8966951171875, 'test/num_examples': 10000, 'score': 36090.768891096115, 'total_duration': 38540.7880191803, 'accumulated_submission_time': 36090.768891096115, 'accumulated_eval_time': 2339.386500597, 'accumulated_logging_time': 5.942543029785156} +I0915 20:46:43.178719 139670985844480 logging_writer.py:48] [44048] accumulated_eval_time=2339.39, accumulated_logging_time=5.94254, accumulated_submission_time=36090.8, global_step=44048, preemption_count=0, score=36090.8, test/accuracy=0.5681, test/loss=1.8967, test/num_examples=10000, total_duration=38540.8, train/accuracy=0.667832, train/loss=2.44587, validation/accuracy=0.5822, validation/loss=2.69446, validation/num_examples=50000 +I0915 20:53:18.662451 139670960666368 logging_writer.py:48] [44500] global_step=44500, grad_norm=0.5, loss=1.78253 +I0915 20:53:18.666177 139691287286976 submission.py:307] 44500) loss = 1.783, grad_norm = 0.500 +I0915 21:02:41.150004 139670985844480 logging_writer.py:48] [45000] global_step=45000, grad_norm=0.5, loss=1.73024 +I0915 21:02:41.154010 139691287286976 submission.py:307] 45000) loss = 1.730, grad_norm = 0.500 +I0915 21:07:15.237610 139670960666368 logging_writer.py:48] [45500] global_step=45500, grad_norm=0.5, loss=1.82686 +I0915 21:07:15.241750 139691287286976 submission.py:307] 45500) loss = 1.827, grad_norm = 0.500 +I0915 21:14:41.677797 139670985844480 logging_writer.py:48] [46000] global_step=46000, grad_norm=0.5, loss=1.74022 +I0915 21:14:41.683088 139691287286976 submission.py:307] 46000) loss = 1.740, grad_norm = 0.500 +I0915 21:21:25.894186 139670960666368 logging_writer.py:48] [46500] global_step=46500, grad_norm=0.5, loss=1.86033 +I0915 21:21:25.898704 139691287286976 submission.py:307] 46500) loss = 1.860, grad_norm = 0.500 +I0915 21:26:39.093281 139670985844480 logging_writer.py:48] [47000] global_step=47000, grad_norm=0.5, loss=2.00655 +I0915 21:26:39.097467 139691287286976 submission.py:307] 47000) loss = 2.007, grad_norm = 0.500 +I0915 21:29:42.693772 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 21:31:06.328704 139691287286976 spec.py:346] Evaluating on the validation split. +I0915 21:32:12.656740 139691287286976 spec.py:363] Evaluating on the test split. +I0915 21:32:13.658696 139691287286976 submission_runner.py:516] Time since start: 41271.61s, Step: 47187, {'train/accuracy': 0.61841796875, 'train/loss': 3.3203265380859377, 'validation/accuracy': 0.58706, 'validation/loss': 2.7905684375, 'validation/num_examples': 50000, 'test/accuracy': 0.5785, 'test/loss': 1.8651451171875, 'test/num_examples': 10000, 'score': 38661.46665644646, 'total_duration': 41271.61074280739, 'accumulated_submission_time': 38661.46665644646, 'accumulated_eval_time': 2490.3515722751617, 'accumulated_logging_time': 6.298062562942505} +I0915 21:32:14.026180 139670952273664 logging_writer.py:48] [47187] accumulated_eval_time=2490.35, accumulated_logging_time=6.29806, accumulated_submission_time=38661.5, global_step=47187, preemption_count=0, score=38661.5, test/accuracy=0.5785, test/loss=1.86515, test/num_examples=10000, total_duration=41271.6, train/accuracy=0.618418, train/loss=3.32033, validation/accuracy=0.58706, validation/loss=2.79057, validation/num_examples=50000 +I0915 21:37:01.565543 139671078164224 logging_writer.py:48] [47500] global_step=47500, grad_norm=0.5, loss=3.18195 +I0915 21:37:01.571071 139691287286976 submission.py:307] 47500) loss = 3.182, grad_norm = 0.500 +I0915 21:42:13.095861 139670952273664 logging_writer.py:48] [48000] global_step=48000, grad_norm=0.499999, loss=2.26477 +I0915 21:42:13.101833 139691287286976 submission.py:307] 48000) loss = 2.265, grad_norm = 0.500 +I0915 21:49:33.094033 139671078164224 logging_writer.py:48] [48500] global_step=48500, grad_norm=0.5, loss=1.69721 +I0915 21:49:33.099227 139691287286976 submission.py:307] 48500) loss = 1.697, grad_norm = 0.500 +I0915 21:56:11.049960 139670952273664 logging_writer.py:48] [49000] global_step=49000, grad_norm=0.5, loss=1.61269 +I0915 21:56:11.054445 139691287286976 submission.py:307] 49000) loss = 1.613, grad_norm = 0.500 +I0915 22:01:32.092499 139671078164224 logging_writer.py:48] [49500] global_step=49500, grad_norm=0.5, loss=3.77043 +I0915 22:01:32.097264 139691287286976 submission.py:307] 49500) loss = 3.770, grad_norm = 0.500 +I0915 22:10:22.009984 139670952273664 logging_writer.py:48] [50000] global_step=50000, grad_norm=0.5, loss=1.57802 +I0915 22:10:22.015367 139691287286976 submission.py:307] 50000) loss = 1.578, grad_norm = 0.500 +I0915 22:14:47.462286 139671078164224 logging_writer.py:48] [50500] global_step=50500, grad_norm=0.5, loss=1.67948 +I0915 22:14:47.485968 139691287286976 submission.py:307] 50500) loss = 1.679, grad_norm = 0.500 +I0915 22:15:09.425100 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 22:16:33.279108 139691287286976 spec.py:346] Evaluating on the validation split. +I0915 22:17:39.094654 139691287286976 spec.py:363] Evaluating on the test split. +I0915 22:17:40.007873 139691287286976 submission_runner.py:516] Time since start: 43997.96s, Step: 50538, {'train/accuracy': 0.663828125, 'train/loss': 2.6336328125, 'validation/accuracy': 0.59092, 'validation/loss': 2.7246275, 'validation/num_examples': 50000, 'test/accuracy': 0.5798, 'test/loss': 1.82652265625, 'test/num_examples': 10000, 'score': 41228.18127465248, 'total_duration': 43997.95974302292, 'accumulated_submission_time': 41228.18127465248, 'accumulated_eval_time': 2640.9342324733734, 'accumulated_logging_time': 6.681631803512573} +I0915 22:17:40.321328 139671086556928 logging_writer.py:48] [50538] accumulated_eval_time=2640.93, accumulated_logging_time=6.68163, accumulated_submission_time=41228.2, global_step=50538, preemption_count=0, score=41228.2, test/accuracy=0.5798, test/loss=1.82652, test/num_examples=10000, total_duration=43998, train/accuracy=0.663828, train/loss=2.63363, validation/accuracy=0.59092, validation/loss=2.72463, validation/num_examples=50000 +I0915 22:24:44.515707 139670994237184 logging_writer.py:48] [51000] global_step=51000, grad_norm=0.5, loss=1.64569 +I0915 22:24:44.519507 139691287286976 submission.py:307] 51000) loss = 1.646, grad_norm = 0.500 +I0915 22:31:44.527633 139671086556928 logging_writer.py:48] [51500] global_step=51500, grad_norm=0.5, loss=1.65082 +I0915 22:31:44.539964 139691287286976 submission.py:307] 51500) loss = 1.651, grad_norm = 0.500 +I0915 22:37:06.713808 139670994237184 logging_writer.py:48] [52000] global_step=52000, grad_norm=0.5, loss=1.75336 +I0915 22:37:06.719812 139691287286976 submission.py:307] 52000) loss = 1.753, grad_norm = 0.500 +I0915 22:46:06.294197 139671086556928 logging_writer.py:48] [52500] global_step=52500, grad_norm=0.5, loss=1.67138 +I0915 22:46:06.299576 139691287286976 submission.py:307] 52500) loss = 1.671, grad_norm = 0.500 +I0915 22:50:30.525540 139670994237184 logging_writer.py:48] [53000] global_step=53000, grad_norm=0.5, loss=4.212 +I0915 22:50:30.529886 139691287286976 submission.py:307] 53000) loss = 4.212, grad_norm = 0.500 +I0915 22:57:22.157259 139671086556928 logging_writer.py:48] [53500] global_step=53500, grad_norm=0.5, loss=3.20865 +I0915 22:57:22.162340 139691287286976 submission.py:307] 53500) loss = 3.209, grad_norm = 0.500 +I0915 23:00:39.409354 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 23:02:10.600182 139691287286976 spec.py:346] Evaluating on the validation split. +I0915 23:03:16.126122 139691287286976 spec.py:363] Evaluating on the test split. +I0915 23:03:17.063795 139691287286976 submission_runner.py:516] Time since start: 46735.02s, Step: 53663, {'train/accuracy': 0.66783203125, 'train/loss': 2.7169110107421877, 'validation/accuracy': 0.6055, 'validation/loss': 2.58237921875, 'validation/num_examples': 50000, 'test/accuracy': 0.587, 'test/loss': 1.7989046875, 'test/num_examples': 10000, 'score': 43798.88992142677, 'total_duration': 46735.015983104706, 'accumulated_submission_time': 43798.88992142677, 'accumulated_eval_time': 2798.588620901108, 'accumulated_logging_time': 7.0068535804748535} +I0915 23:03:17.346871 139671044593408 logging_writer.py:48] [53663] accumulated_eval_time=2798.59, accumulated_logging_time=7.00685, accumulated_submission_time=43798.9, global_step=53663, preemption_count=0, score=43798.9, test/accuracy=0.587, test/loss=1.7989, test/num_examples=10000, total_duration=46735, train/accuracy=0.667832, train/loss=2.71691, validation/accuracy=0.6055, validation/loss=2.58238, validation/num_examples=50000 +I0915 23:07:09.211760 139670952273664 logging_writer.py:48] [54000] global_step=54000, grad_norm=0.5, loss=2.46787 +I0915 23:07:09.215642 139691287286976 submission.py:307] 54000) loss = 2.468, grad_norm = 0.500 +I0915 23:13:43.308104 139671044593408 logging_writer.py:48] [54500] global_step=54500, grad_norm=0.5, loss=2.04261 +I0915 23:13:43.311988 139691287286976 submission.py:307] 54500) loss = 2.043, grad_norm = 0.500 +I0915 23:22:39.030120 139670952273664 logging_writer.py:48] [55000] global_step=55000, grad_norm=0.5, loss=2.78407 +I0915 23:22:39.034134 139691287286976 submission.py:307] 55000) loss = 2.784, grad_norm = 0.500 +I0915 23:26:54.818115 139671044593408 logging_writer.py:48] [55500] global_step=55500, grad_norm=0.5, loss=3.38585 +I0915 23:26:54.822332 139691287286976 submission.py:307] 55500) loss = 3.386, grad_norm = 0.500 +I0915 23:34:21.267112 139670952273664 logging_writer.py:48] [56000] global_step=56000, grad_norm=0.5, loss=1.7718 +I0915 23:34:21.271124 139691287286976 submission.py:307] 56000) loss = 1.772, grad_norm = 0.500 +I0915 23:41:23.243322 139671044593408 logging_writer.py:48] [56500] global_step=56500, grad_norm=0.5, loss=2.80602 +I0915 23:41:23.248726 139691287286976 submission.py:307] 56500) loss = 2.806, grad_norm = 0.500 +I0915 23:46:20.035340 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0915 23:47:41.060981 139691287286976 spec.py:346] Evaluating on the validation split. +I0915 23:48:47.464679 139691287286976 spec.py:363] Evaluating on the test split. +I0915 23:48:48.500677 139691287286976 submission_runner.py:516] Time since start: 49466.45s, Step: 56977, {'train/accuracy': 0.67353515625, 'train/loss': 2.700517578125, 'validation/accuracy': 0.6015, 'validation/loss': 2.7015803125, 'validation/num_examples': 50000, 'test/accuracy': 0.596, 'test/loss': 1.776692578125, 'test/num_examples': 10000, 'score': 46373.18920493126, 'total_duration': 49466.45277762413, 'accumulated_submission_time': 46373.18920493126, 'accumulated_eval_time': 2947.053974866867, 'accumulated_logging_time': 7.3032426834106445} +I0915 23:48:48.853379 139671061378816 logging_writer.py:48] [56977] accumulated_eval_time=2947.05, accumulated_logging_time=7.30324, accumulated_submission_time=46373.2, global_step=56977, preemption_count=0, score=46373.2, test/accuracy=0.596, test/loss=1.77669, test/num_examples=10000, total_duration=49466.5, train/accuracy=0.673535, train/loss=2.70052, validation/accuracy=0.6015, validation/loss=2.70158, validation/num_examples=50000 +I0915 23:48:55.903020 139670943880960 logging_writer.py:48] [57000] global_step=57000, grad_norm=0.5, loss=1.64529 +I0915 23:48:55.908882 139691287286976 submission.py:307] 57000) loss = 1.645, grad_norm = 0.500 +I0915 23:57:21.624549 139671061378816 logging_writer.py:48] [57500] global_step=57500, grad_norm=0.5, loss=3.18012 +I0915 23:57:21.628527 139691287286976 submission.py:307] 57500) loss = 3.180, grad_norm = 0.500 +I0916 00:02:18.436025 139670943880960 logging_writer.py:48] [58000] global_step=58000, grad_norm=0.5, loss=3.95889 +I0916 00:02:18.443732 139691287286976 submission.py:307] 58000) loss = 3.959, grad_norm = 0.500 +I0916 00:09:42.381192 139671061378816 logging_writer.py:48] [58500] global_step=58500, grad_norm=0.5, loss=2.36117 +I0916 00:09:42.385360 139691287286976 submission.py:307] 58500) loss = 2.361, grad_norm = 0.500 +I0916 00:16:34.047819 139670943880960 logging_writer.py:48] [59000] global_step=59000, grad_norm=0.5, loss=1.73557 +I0916 00:16:34.069907 139691287286976 submission.py:307] 59000) loss = 1.736, grad_norm = 0.500 +I0916 00:21:54.446543 139671061378816 logging_writer.py:48] [59500] global_step=59500, grad_norm=0.5, loss=2.98648 +I0916 00:21:54.454769 139691287286976 submission.py:307] 59500) loss = 2.986, grad_norm = 0.500 +I0916 00:30:30.577564 139670943880960 logging_writer.py:48] [60000] global_step=60000, grad_norm=0.5, loss=3.90712 +I0916 00:30:30.582915 139691287286976 submission.py:307] 60000) loss = 3.907, grad_norm = 0.500 +I0916 00:31:47.165896 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 00:32:58.257508 139691287286976 spec.py:346] Evaluating on the validation split. +I0916 00:34:04.570890 139691287286976 spec.py:363] Evaluating on the test split. +I0916 00:34:05.505600 139691287286976 submission_runner.py:516] Time since start: 52183.46s, Step: 60128, {'train/accuracy': 0.71884765625, 'train/loss': 2.027147674560547, 'validation/accuracy': 0.6182, 'validation/loss': 2.40730015625, 'validation/num_examples': 50000, 'test/accuracy': 0.5981, 'test/loss': 1.76907109375, 'test/num_examples': 10000, 'score': 48942.8839867115, 'total_duration': 52183.45779824257, 'accumulated_submission_time': 48942.8839867115, 'accumulated_eval_time': 3085.3943016529083, 'accumulated_logging_time': 7.672607660293579} +I0916 00:34:05.829197 139671052986112 logging_writer.py:48] [60128] accumulated_eval_time=3085.39, accumulated_logging_time=7.67261, accumulated_submission_time=48942.9, global_step=60128, preemption_count=0, score=48942.9, test/accuracy=0.5981, test/loss=1.76907, test/num_examples=10000, total_duration=52183.5, train/accuracy=0.718848, train/loss=2.02715, validation/accuracy=0.6182, validation/loss=2.4073, validation/num_examples=50000 +I0916 00:38:57.006893 139670969059072 logging_writer.py:48] [60500] global_step=60500, grad_norm=0.5, loss=1.72743 +I0916 00:38:57.012873 139691287286976 submission.py:307] 60500) loss = 1.727, grad_norm = 0.500 +I0916 00:47:23.034822 139671052986112 logging_writer.py:48] [61000] global_step=61000, grad_norm=0.5, loss=1.82134 +I0916 00:47:23.040795 139691287286976 submission.py:307] 61000) loss = 1.821, grad_norm = 0.500 +I0916 00:54:30.443982 139670969059072 logging_writer.py:48] [61500] global_step=61500, grad_norm=0.5, loss=3.18039 +I0916 00:54:30.478710 139691287286976 submission.py:307] 61500) loss = 3.180, grad_norm = 0.500 +I0916 00:59:35.927430 139671052986112 logging_writer.py:48] [62000] global_step=62000, grad_norm=0.5, loss=1.56838 +I0916 00:59:35.932600 139691287286976 submission.py:307] 62000) loss = 1.568, grad_norm = 0.500 +I0916 01:08:48.202996 139670969059072 logging_writer.py:48] [62500] global_step=62500, grad_norm=0.5, loss=1.99063 +I0916 01:08:48.207281 139691287286976 submission.py:307] 62500) loss = 1.991, grad_norm = 0.500 +I0916 01:13:23.743624 139671052986112 logging_writer.py:48] [63000] global_step=63000, grad_norm=0.5, loss=1.37847 +I0916 01:13:23.749104 139691287286976 submission.py:307] 63000) loss = 1.378, grad_norm = 0.500 +I0916 01:17:01.514787 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 01:18:26.006687 139691287286976 spec.py:346] Evaluating on the validation split. +I0916 01:19:31.666695 139691287286976 spec.py:363] Evaluating on the test split. +I0916 01:19:32.598239 139691287286976 submission_runner.py:516] Time since start: 54910.55s, Step: 63303, {'train/accuracy': 0.68916015625, 'train/loss': 2.5399473571777342, 'validation/accuracy': 0.6227, 'validation/loss': 2.3859615625, 'validation/num_examples': 50000, 'test/accuracy': 0.6045, 'test/loss': 1.758034765625, 'test/num_examples': 10000, 'score': 51509.77036523819, 'total_duration': 54910.55043578148, 'accumulated_submission_time': 51509.77036523819, 'accumulated_eval_time': 3236.4778270721436, 'accumulated_logging_time': 8.006976842880249} +I0916 01:19:32.871548 139671044593408 logging_writer.py:48] [63303] accumulated_eval_time=3236.48, accumulated_logging_time=8.00698, accumulated_submission_time=51509.8, global_step=63303, preemption_count=0, score=51509.8, test/accuracy=0.6045, test/loss=1.75803, test/num_examples=10000, total_duration=54910.6, train/accuracy=0.68916, train/loss=2.53995, validation/accuracy=0.6227, validation/loss=2.38596, validation/num_examples=50000 +I0916 01:21:54.578018 139671002629888 logging_writer.py:48] [63500] global_step=63500, grad_norm=0.5, loss=1.37863 +I0916 01:21:54.582883 139691287286976 submission.py:307] 63500) loss = 1.379, grad_norm = 0.500 +I0916 01:29:20.783231 139671044593408 logging_writer.py:48] [64000] global_step=64000, grad_norm=0.5, loss=1.41371 +I0916 01:29:20.788050 139691287286976 submission.py:307] 64000) loss = 1.414, grad_norm = 0.500 +I0916 01:34:51.600718 139671002629888 logging_writer.py:48] [64500] global_step=64500, grad_norm=0.5, loss=1.64764 +I0916 01:34:51.605489 139691287286976 submission.py:307] 64500) loss = 1.648, grad_norm = 0.500 +I0916 01:43:37.531061 139671044593408 logging_writer.py:48] [65000] global_step=65000, grad_norm=0.5, loss=1.97741 +I0916 01:43:37.535070 139691287286976 submission.py:307] 65000) loss = 1.977, grad_norm = 0.500 +I0916 01:48:14.114971 139671002629888 logging_writer.py:48] [65500] global_step=65500, grad_norm=0.5, loss=2.51921 +I0916 01:48:14.119185 139691287286976 submission.py:307] 65500) loss = 2.519, grad_norm = 0.500 +I0916 01:55:21.110334 139671044593408 logging_writer.py:48] [66000] global_step=66000, grad_norm=0.5, loss=3.12234 +I0916 01:55:21.114246 139691287286976 submission.py:307] 66000) loss = 3.122, grad_norm = 0.500 +I0916 02:02:05.339301 139671002629888 logging_writer.py:48] [66500] global_step=66500, grad_norm=0.5, loss=1.47966 +I0916 02:02:05.367897 139691287286976 submission.py:307] 66500) loss = 1.480, grad_norm = 0.500 +I0916 02:02:32.921473 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 02:03:47.640341 139691287286976 spec.py:346] Evaluating on the validation split. +I0916 02:04:53.466132 139691287286976 spec.py:363] Evaluating on the test split. +I0916 02:04:54.370254 139691287286976 submission_runner.py:516] Time since start: 57632.32s, Step: 66544, {'train/accuracy': 0.67556640625, 'train/loss': 2.814586181640625, 'validation/accuracy': 0.61188, 'validation/loss': 2.5821090625, 'validation/num_examples': 50000, 'test/accuracy': 0.6037, 'test/loss': 1.736313671875, 'test/num_examples': 10000, 'score': 54081.4677426815, 'total_duration': 57632.322525024414, 'accumulated_submission_time': 54081.4677426815, 'accumulated_eval_time': 3377.9268865585327, 'accumulated_logging_time': 8.292526960372925} +I0916 02:04:54.665446 139671078164224 logging_writer.py:48] [66544] accumulated_eval_time=3377.93, accumulated_logging_time=8.29253, accumulated_submission_time=54081.5, global_step=66544, preemption_count=0, score=54081.5, test/accuracy=0.6037, test/loss=1.73631, test/num_examples=10000, total_duration=57632.3, train/accuracy=0.675566, train/loss=2.81459, validation/accuracy=0.61188, validation/loss=2.58211, validation/num_examples=50000 +I0916 02:11:34.663202 139671011022592 logging_writer.py:48] [67000] global_step=67000, grad_norm=0.5, loss=2.23558 +I0916 02:11:34.668164 139691287286976 submission.py:307] 67000) loss = 2.236, grad_norm = 0.500 +I0916 02:20:58.546733 139671078164224 logging_writer.py:48] [67500] global_step=67500, grad_norm=0.5, loss=2.69299 +I0916 02:20:58.552020 139691287286976 submission.py:307] 67500) loss = 2.693, grad_norm = 0.500 +I0916 02:25:39.327621 139671011022592 logging_writer.py:48] [68000] global_step=68000, grad_norm=0.499999, loss=2.72269 +I0916 02:25:39.331887 139691287286976 submission.py:307] 68000) loss = 2.723, grad_norm = 0.500 +I0916 02:32:53.753441 139671078164224 logging_writer.py:48] [68500] global_step=68500, grad_norm=0.5, loss=4.02516 +I0916 02:32:53.758663 139691287286976 submission.py:307] 68500) loss = 4.025, grad_norm = 0.500 +I0916 02:40:04.778357 139671011022592 logging_writer.py:48] [69000] global_step=69000, grad_norm=0.5, loss=1.51283 +I0916 02:40:04.785193 139691287286976 submission.py:307] 69000) loss = 1.513, grad_norm = 0.500 +I0916 02:45:12.824245 139671078164224 logging_writer.py:48] [69500] global_step=69500, grad_norm=0.5, loss=1.53944 +I0916 02:45:12.829726 139691287286976 submission.py:307] 69500) loss = 1.539, grad_norm = 0.500 +I0916 02:47:51.016111 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 02:49:14.642708 139691287286976 spec.py:346] Evaluating on the validation split. +I0916 02:50:20.315815 139691287286976 spec.py:363] Evaluating on the test split. +I0916 02:50:21.226956 139691287286976 submission_runner.py:516] Time since start: 60359.18s, Step: 69665, {'train/accuracy': 0.69140625, 'train/loss': 2.551528015136719, 'validation/accuracy': 0.61948, 'validation/loss': 2.4735353125, 'validation/num_examples': 50000, 'test/accuracy': 0.6067, 'test/loss': 1.7296421875, 'test/num_examples': 10000, 'score': 56649.6544585228, 'total_duration': 60359.17927026749, 'accumulated_submission_time': 56649.6544585228, 'accumulated_eval_time': 3528.137953519821, 'accumulated_logging_time': 8.600436449050903} +I0916 02:50:21.575015 139670952273664 logging_writer.py:48] [69665] accumulated_eval_time=3528.14, accumulated_logging_time=8.60044, accumulated_submission_time=56649.7, global_step=69665, preemption_count=0, score=56649.7, test/accuracy=0.6067, test/loss=1.72964, test/num_examples=10000, total_duration=60359.2, train/accuracy=0.691406, train/loss=2.55153, validation/accuracy=0.61948, validation/loss=2.47354, validation/num_examples=50000 +I0916 02:55:33.526181 139671002629888 logging_writer.py:48] [70000] global_step=70000, grad_norm=0.5, loss=1.55032 +I0916 02:55:33.531047 139691287286976 submission.py:307] 70000) loss = 1.550, grad_norm = 0.500 +I0916 03:00:46.272233 139670952273664 logging_writer.py:48] [70500] global_step=70500, grad_norm=0.5, loss=1.41922 +I0916 03:00:46.276774 139691287286976 submission.py:307] 70500) loss = 1.419, grad_norm = 0.500 +I0916 03:07:56.711551 139671002629888 logging_writer.py:48] [71000] global_step=71000, grad_norm=0.5, loss=2.04243 +I0916 03:07:56.715417 139691287286976 submission.py:307] 71000) loss = 2.042, grad_norm = 0.500 +I0916 03:14:43.051029 139670952273664 logging_writer.py:48] [71500] global_step=71500, grad_norm=0.5, loss=1.71343 +I0916 03:14:43.057321 139691287286976 submission.py:307] 71500) loss = 1.713, grad_norm = 0.500 +I0916 03:19:51.358991 139671002629888 logging_writer.py:48] [72000] global_step=72000, grad_norm=0.5, loss=3.0458 +I0916 03:19:51.363076 139691287286976 submission.py:307] 72000) loss = 3.046, grad_norm = 0.500 +I0916 03:28:33.786206 139670952273664 logging_writer.py:48] [72500] global_step=72500, grad_norm=0.5, loss=1.38827 +I0916 03:28:33.791640 139691287286976 submission.py:307] 72500) loss = 1.388, grad_norm = 0.500 +I0916 03:32:59.483704 139671002629888 logging_writer.py:48] [73000] global_step=73000, grad_norm=0.5, loss=2.00527 +I0916 03:32:59.503382 139691287286976 submission.py:307] 73000) loss = 2.005, grad_norm = 0.500 +I0916 03:33:18.402332 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 03:34:35.337360 139691287286976 spec.py:346] Evaluating on the validation split. +I0916 03:35:41.145884 139691287286976 spec.py:363] Evaluating on the test split. +I0916 03:35:42.064301 139691287286976 submission_runner.py:516] Time since start: 63080.02s, Step: 73022, {'train/accuracy': 0.713828125, 'train/loss': 2.382465362548828, 'validation/accuracy': 0.6335, 'validation/loss': 2.30428890625, 'validation/num_examples': 50000, 'test/accuracy': 0.6071, 'test/loss': 1.7324955078125, 'test/num_examples': 10000, 'score': 59218.12683725357, 'total_duration': 63080.01647400856, 'accumulated_submission_time': 59218.12683725357, 'accumulated_eval_time': 3671.8004858493805, 'accumulated_logging_time': 8.961181879043579} +I0916 03:35:42.361592 139671052986112 logging_writer.py:48] [73022] accumulated_eval_time=3671.8, accumulated_logging_time=8.96118, accumulated_submission_time=59218.1, global_step=73022, preemption_count=0, score=59218.1, test/accuracy=0.6071, test/loss=1.7325, test/num_examples=10000, total_duration=63080, train/accuracy=0.713828, train/loss=2.38247, validation/accuracy=0.6335, validation/loss=2.30429, validation/num_examples=50000 +I0916 03:42:50.521540 139670943880960 logging_writer.py:48] [73500] global_step=73500, grad_norm=0.5, loss=1.39458 +I0916 03:42:50.527079 139691287286976 submission.py:307] 73500) loss = 1.395, grad_norm = 0.500 +I0916 03:50:13.345144 139671052986112 logging_writer.py:48] [74000] global_step=74000, grad_norm=0.5, loss=1.32348 +I0916 03:50:13.353594 139691287286976 submission.py:307] 74000) loss = 1.323, grad_norm = 0.500 +I0916 03:55:23.152293 139670943880960 logging_writer.py:48] [74500] global_step=74500, grad_norm=0.5, loss=3.5307 +I0916 03:55:23.156423 139691287286976 submission.py:307] 74500) loss = 3.531, grad_norm = 0.500 +I0916 04:04:19.891350 139671052986112 logging_writer.py:48] [75000] global_step=75000, grad_norm=0.5, loss=2.31899 +I0916 04:04:19.895311 139691287286976 submission.py:307] 75000) loss = 2.319, grad_norm = 0.500 +I0916 04:08:58.378216 139670943880960 logging_writer.py:48] [75500] global_step=75500, grad_norm=0.5, loss=1.36404 +I0916 04:08:58.382510 139691287286976 submission.py:307] 75500) loss = 1.364, grad_norm = 0.500 +I0916 04:15:45.772984 139671052986112 logging_writer.py:48] [76000] global_step=76000, grad_norm=0.5, loss=1.3471 +I0916 04:15:45.777476 139691287286976 submission.py:307] 76000) loss = 1.347, grad_norm = 0.500 +I0916 04:18:45.488753 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 04:20:18.671630 139691287286976 spec.py:346] Evaluating on the validation split. +I0916 04:21:24.065118 139691287286976 spec.py:363] Evaluating on the test split. +I0916 04:21:24.988716 139691287286976 submission_runner.py:516] Time since start: 65822.94s, Step: 76160, {'train/accuracy': 0.72046875, 'train/loss': 2.2783940124511717, 'validation/accuracy': 0.63344, 'validation/loss': 2.3765715625, 'validation/num_examples': 50000, 'test/accuracy': 0.6167, 'test/loss': 1.67845546875, 'test/num_examples': 10000, 'score': 61793.17218565941, 'total_duration': 65822.94068861008, 'accumulated_submission_time': 61793.17218565941, 'accumulated_eval_time': 3831.3002483844757, 'accumulated_logging_time': 9.277946949005127} +I0916 04:21:25.306208 139670977451776 logging_writer.py:48] [76160] accumulated_eval_time=3831.3, accumulated_logging_time=9.27795, accumulated_submission_time=61793.2, global_step=76160, preemption_count=0, score=61793.2, test/accuracy=0.6167, test/loss=1.67846, test/num_examples=10000, total_duration=65822.9, train/accuracy=0.720469, train/loss=2.27839, validation/accuracy=0.63344, validation/loss=2.37657, validation/num_examples=50000 +I0916 04:25:23.203978 139671094949632 logging_writer.py:48] [76500] global_step=76500, grad_norm=0.5, loss=1.42011 +I0916 04:25:23.215901 139691287286976 submission.py:307] 76500) loss = 1.420, grad_norm = 0.500 +I0916 04:31:47.279502 139670977451776 logging_writer.py:48] [77000] global_step=77000, grad_norm=0.5, loss=2.17526 +I0916 04:31:47.285348 139691287286976 submission.py:307] 77000) loss = 2.175, grad_norm = 0.500 +I0916 04:40:37.594255 139671094949632 logging_writer.py:48] [77500] global_step=77500, grad_norm=0.5, loss=1.87779 +I0916 04:40:37.600083 139691287286976 submission.py:307] 77500) loss = 1.878, grad_norm = 0.500 +I0916 04:44:56.314309 139670977451776 logging_writer.py:48] [78000] global_step=78000, grad_norm=0.5, loss=3.72739 +I0916 04:44:56.318600 139691287286976 submission.py:307] 78000) loss = 3.727, grad_norm = 0.500 +I0916 04:52:08.452499 139671094949632 logging_writer.py:48] [78500] global_step=78500, grad_norm=0.5, loss=3.04677 +I0916 04:52:08.456621 139691287286976 submission.py:307] 78500) loss = 3.047, grad_norm = 0.500 +I0916 04:59:25.604593 139670977451776 logging_writer.py:48] [79000] global_step=79000, grad_norm=0.5, loss=1.40295 +I0916 04:59:25.611283 139691287286976 submission.py:307] 79000) loss = 1.403, grad_norm = 0.500 +I0916 05:04:22.298133 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 05:05:48.076911 139691287286976 spec.py:346] Evaluating on the validation split. +I0916 05:06:53.559192 139691287286976 spec.py:363] Evaluating on the test split. +I0916 05:06:54.535540 139691287286976 submission_runner.py:516] Time since start: 68552.49s, Step: 79490, {'train/accuracy': 0.68509765625, 'train/loss': 2.83866455078125, 'validation/accuracy': 0.61306, 'validation/loss': 2.6923934375, 'validation/num_examples': 50000, 'test/accuracy': 0.6165, 'test/loss': 1.7130814453125, 'test/num_examples': 10000, 'score': 64361.67498803139, 'total_duration': 68552.48768758774, 'accumulated_submission_time': 64361.67498803139, 'accumulated_eval_time': 3983.537580013275, 'accumulated_logging_time': 9.606787919998169} +I0916 05:06:54.821326 139670994237184 logging_writer.py:48] [79490] accumulated_eval_time=3983.54, accumulated_logging_time=9.60679, accumulated_submission_time=64361.7, global_step=79490, preemption_count=0, score=64361.7, test/accuracy=0.6165, test/loss=1.71308, test/num_examples=10000, total_duration=68552.5, train/accuracy=0.685098, train/loss=2.83866, validation/accuracy=0.61306, validation/loss=2.69239, validation/num_examples=50000 +I0916 05:06:58.715609 139670943880960 logging_writer.py:48] [79500] global_step=79500, grad_norm=0.5, loss=1.33008 +I0916 05:06:58.720530 139691287286976 submission.py:307] 79500) loss = 1.330, grad_norm = 0.500 +I0916 05:15:10.827786 139670994237184 logging_writer.py:48] [80000] global_step=80000, grad_norm=0.5, loss=1.54495 +I0916 05:15:10.834450 139691287286976 submission.py:307] 80000) loss = 1.545, grad_norm = 0.500 +I0916 05:20:18.769044 139670943880960 logging_writer.py:48] [80500] global_step=80500, grad_norm=0.5, loss=1.59107 +I0916 05:20:18.773009 139691287286976 submission.py:307] 80500) loss = 1.591, grad_norm = 0.500 +I0916 05:27:23.547426 139670994237184 logging_writer.py:48] [81000] global_step=81000, grad_norm=0.5, loss=2.15787 +I0916 05:27:23.553355 139691287286976 submission.py:307] 81000) loss = 2.158, grad_norm = 0.500 +I0916 05:34:30.698923 139670943880960 logging_writer.py:48] [81500] global_step=81500, grad_norm=0.5, loss=1.33292 +I0916 05:34:30.702992 139691287286976 submission.py:307] 81500) loss = 1.333, grad_norm = 0.500 +I0916 05:39:34.400881 139670994237184 logging_writer.py:48] [82000] global_step=82000, grad_norm=0.5, loss=2.31208 +I0916 05:39:34.407242 139691287286976 submission.py:307] 82000) loss = 2.312, grad_norm = 0.500 +I0916 05:48:10.182284 139670943880960 logging_writer.py:48] [82500] global_step=82500, grad_norm=0.5, loss=1.22216 +I0916 05:48:10.186491 139691287286976 submission.py:307] 82500) loss = 1.222, grad_norm = 0.500 +I0916 05:49:54.314534 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 05:51:04.780244 139691287286976 spec.py:346] Evaluating on the validation split. +I0916 05:52:10.189928 139691287286976 spec.py:363] Evaluating on the test split. +I0916 05:52:11.212890 139691287286976 submission_runner.py:516] Time since start: 71269.16s, Step: 82666, {'train/accuracy': 0.73498046875, 'train/loss': 2.2339772033691405, 'validation/accuracy': 0.63742, 'validation/loss': 2.43264578125, 'validation/num_examples': 50000, 'test/accuracy': 0.6212, 'test/loss': 1.6881970703125, 'test/num_examples': 10000, 'score': 66933.0193309784, 'total_duration': 71269.16496658325, 'accumulated_submission_time': 66933.0193309784, 'accumulated_eval_time': 4120.435905694962, 'accumulated_logging_time': 9.923115968704224} +I0916 05:52:11.651139 139670952273664 logging_writer.py:48] [82666] accumulated_eval_time=4120.44, accumulated_logging_time=9.92312, accumulated_submission_time=66933, global_step=82666, preemption_count=0, score=66933, test/accuracy=0.6212, test/loss=1.6882, test/num_examples=10000, total_duration=71269.2, train/accuracy=0.73498, train/loss=2.23398, validation/accuracy=0.63742, validation/loss=2.43265, validation/num_examples=50000 +I0916 05:56:25.756948 139671925417728 logging_writer.py:48] [83000] global_step=83000, grad_norm=0.5, loss=1.35706 +I0916 05:56:25.760950 139691287286976 submission.py:307] 83000) loss = 1.357, grad_norm = 0.500 +I0916 06:04:47.237624 139670952273664 logging_writer.py:48] [83500] global_step=83500, grad_norm=0.5, loss=1.33902 +I0916 06:04:47.243106 139691287286976 submission.py:307] 83500) loss = 1.339, grad_norm = 0.500 +I0916 06:12:16.995337 139671925417728 logging_writer.py:48] [84000] global_step=84000, grad_norm=0.5, loss=1.39249 +I0916 06:12:16.999650 139691287286976 submission.py:307] 84000) loss = 1.392, grad_norm = 0.500 +I0916 06:17:15.287943 139670952273664 logging_writer.py:48] [84500] global_step=84500, grad_norm=0.5, loss=3.59766 +I0916 06:17:15.293278 139691287286976 submission.py:307] 84500) loss = 3.598, grad_norm = 0.500 +I0916 06:26:21.769306 139671925417728 logging_writer.py:48] [85000] global_step=85000, grad_norm=0.5, loss=2.02725 +I0916 06:26:21.775943 139691287286976 submission.py:307] 85000) loss = 2.027, grad_norm = 0.500 +I0916 06:31:02.659252 139670952273664 logging_writer.py:48] [85500] global_step=85500, grad_norm=0.5, loss=1.36299 +I0916 06:31:02.677979 139691287286976 submission.py:307] 85500) loss = 1.363, grad_norm = 0.500 +I0916 06:35:11.710207 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 06:36:30.321405 139691287286976 spec.py:346] Evaluating on the validation split. +I0916 06:37:36.638120 139691287286976 spec.py:363] Evaluating on the test split. +I0916 06:37:37.585523 139691287286976 submission_runner.py:516] Time since start: 73995.54s, Step: 85838, {'train/accuracy': 0.7084765625, 'train/loss': 2.568540344238281, 'validation/accuracy': 0.61908, 'validation/loss': 2.64155625, 'validation/num_examples': 50000, 'test/accuracy': 0.6217, 'test/loss': 1.6680759765625, 'test/num_examples': 10000, 'score': 69504.26203966141, 'total_duration': 73995.53771471977, 'accumulated_submission_time': 69504.26203966141, 'accumulated_eval_time': 4266.311295747757, 'accumulated_logging_time': 10.376159906387329} +I0916 06:37:37.984875 139670943880960 logging_writer.py:48] [85838] accumulated_eval_time=4266.31, accumulated_logging_time=10.3762, accumulated_submission_time=69504.3, global_step=85838, preemption_count=0, score=69504.3, test/accuracy=0.6217, test/loss=1.66808, test/num_examples=10000, total_duration=73995.5, train/accuracy=0.708477, train/loss=2.56854, validation/accuracy=0.61908, validation/loss=2.64156, validation/num_examples=50000 +I0916 06:39:30.269729 139671925417728 logging_writer.py:48] [86000] global_step=86000, grad_norm=0.5, loss=1.40245 +I0916 06:39:30.274847 139691287286976 submission.py:307] 86000) loss = 1.402, grad_norm = 0.500 +I0916 06:46:59.355873 139670943880960 logging_writer.py:48] [86500] global_step=86500, grad_norm=0.5, loss=1.2691 +I0916 06:46:59.360968 139691287286976 submission.py:307] 86500) loss = 1.269, grad_norm = 0.500 +I0916 06:52:28.345242 139671925417728 logging_writer.py:48] [87000] global_step=87000, grad_norm=0.5, loss=1.32122 +I0916 06:52:28.351392 139691287286976 submission.py:307] 87000) loss = 1.321, grad_norm = 0.500 +I0916 07:01:19.427670 139670943880960 logging_writer.py:48] [87500] global_step=87500, grad_norm=0.5, loss=1.93818 +I0916 07:01:19.435092 139691287286976 submission.py:307] 87500) loss = 1.938, grad_norm = 0.500 +I0916 07:06:03.253046 139671925417728 logging_writer.py:48] [88000] global_step=88000, grad_norm=0.5, loss=2.25363 +I0916 07:06:03.259251 139691287286976 submission.py:307] 88000) loss = 2.254, grad_norm = 0.500 +I0916 07:12:53.384824 139670943880960 logging_writer.py:48] [88500] global_step=88500, grad_norm=0.5, loss=1.60662 +I0916 07:12:53.390218 139691287286976 submission.py:307] 88500) loss = 1.607, grad_norm = 0.500 +I0916 07:19:52.926041 139671925417728 logging_writer.py:48] [89000] global_step=89000, grad_norm=0.5, loss=1.35418 +I0916 07:19:52.953270 139691287286976 submission.py:307] 89000) loss = 1.354, grad_norm = 0.500 +I0916 07:20:34.656186 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 07:21:49.551025 139691287286976 spec.py:346] Evaluating on the validation split. +I0916 07:22:55.278319 139691287286976 spec.py:363] Evaluating on the test split. +I0916 07:22:56.231110 139691287286976 submission_runner.py:516] Time since start: 76714.18s, Step: 89082, {'train/accuracy': 0.73923828125, 'train/loss': 2.213905029296875, 'validation/accuracy': 0.62456, 'validation/loss': 2.56997015625, 'validation/num_examples': 50000, 'test/accuracy': 0.6228, 'test/loss': 1.673147265625, 'test/num_examples': 10000, 'score': 72072.62209844589, 'total_duration': 76714.18280935287, 'accumulated_submission_time': 72072.62209844589, 'accumulated_eval_time': 4407.885825872421, 'accumulated_logging_time': 10.789963960647583} +I0916 07:22:56.705844 139670952273664 logging_writer.py:48] [89082] accumulated_eval_time=4407.89, accumulated_logging_time=10.79, accumulated_submission_time=72072.6, global_step=89082, preemption_count=0, score=72072.6, test/accuracy=0.6228, test/loss=1.67315, test/num_examples=10000, total_duration=76714.2, train/accuracy=0.739238, train/loss=2.21391, validation/accuracy=0.62456, validation/loss=2.56997, validation/num_examples=50000 +I0916 07:28:55.991205 139671036200704 logging_writer.py:48] [89500] global_step=89500, grad_norm=0.5, loss=1.29251 +I0916 07:28:55.996467 139691287286976 submission.py:307] 89500) loss = 1.293, grad_norm = 0.500 +I0916 07:38:18.033279 139670952273664 logging_writer.py:48] [90000] global_step=90000, grad_norm=0.5, loss=1.94434 +I0916 07:38:18.039176 139691287286976 submission.py:307] 90000) loss = 1.944, grad_norm = 0.500 +I0916 07:43:06.747676 139671036200704 logging_writer.py:48] [90500] global_step=90500, grad_norm=0.5, loss=1.30075 +I0916 07:43:06.752832 139691287286976 submission.py:307] 90500) loss = 1.301, grad_norm = 0.500 +I0916 07:50:09.611150 139670952273664 logging_writer.py:48] [91000] global_step=91000, grad_norm=0.5, loss=1.37135 +I0916 07:50:09.615247 139691287286976 submission.py:307] 91000) loss = 1.371, grad_norm = 0.500 +I0916 07:57:28.139320 139671036200704 logging_writer.py:48] [91500] global_step=91500, grad_norm=0.5, loss=3.69932 +I0916 07:57:28.154071 139691287286976 submission.py:307] 91500) loss = 3.699, grad_norm = 0.500 +I0916 08:02:25.851108 139670952273664 logging_writer.py:48] [92000] global_step=92000, grad_norm=0.5, loss=1.30967 +I0916 08:02:25.857277 139691287286976 submission.py:307] 92000) loss = 1.310, grad_norm = 0.500 +I0916 08:05:56.257269 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 08:07:26.290490 139691287286976 spec.py:346] Evaluating on the validation split. +I0916 08:08:32.611912 139691287286976 spec.py:363] Evaluating on the test split. +I0916 08:08:33.547885 139691287286976 submission_runner.py:516] Time since start: 79451.50s, Step: 92224, {'train/accuracy': 0.72103515625, 'train/loss': 2.4955255126953126, 'validation/accuracy': 0.63528, 'validation/loss': 2.4236353125, 'validation/num_examples': 50000, 'test/accuracy': 0.6227, 'test/loss': 1.670181640625, 'test/num_examples': 10000, 'score': 74643.82851338387, 'total_duration': 79451.50010681152, 'accumulated_submission_time': 74643.82851338387, 'accumulated_eval_time': 4565.176446676254, 'accumulated_logging_time': 11.278878450393677} +I0916 08:08:33.831061 139670985844480 logging_writer.py:48] [92224] accumulated_eval_time=4565.18, accumulated_logging_time=11.2789, accumulated_submission_time=74643.8, global_step=92224, preemption_count=0, score=74643.8, test/accuracy=0.6227, test/loss=1.67018, test/num_examples=10000, total_duration=79451.5, train/accuracy=0.721035, train/loss=2.49553, validation/accuracy=0.63528, validation/loss=2.42364, validation/num_examples=50000 +I0916 08:12:34.061008 139670943880960 logging_writer.py:48] [92500] global_step=92500, grad_norm=0.5, loss=2.00386 +I0916 08:12:34.064984 139691287286976 submission.py:307] 92500) loss = 2.004, grad_norm = 0.500 +I0916 08:18:00.586773 139670985844480 logging_writer.py:48] [93000] global_step=93000, grad_norm=0.5, loss=2.41538 +I0916 08:18:00.603698 139691287286976 submission.py:307] 93000) loss = 2.415, grad_norm = 0.500 +I0916 08:25:02.314332 139670943880960 logging_writer.py:48] [93500] global_step=93500, grad_norm=0.5, loss=2.76559 +I0916 08:25:02.318822 139691287286976 submission.py:307] 93500) loss = 2.766, grad_norm = 0.500 +I0916 08:31:59.612544 139670985844480 logging_writer.py:48] [94000] global_step=94000, grad_norm=0.5, loss=1.2334 +I0916 08:31:59.621964 139691287286976 submission.py:307] 94000) loss = 1.233, grad_norm = 0.500 +I0916 08:37:07.601459 139670943880960 logging_writer.py:48] [94500] global_step=94500, grad_norm=0.5, loss=2.88365 +I0916 08:37:07.607554 139691287286976 submission.py:307] 94500) loss = 2.884, grad_norm = 0.500 +I0916 08:45:40.934521 139670985844480 logging_writer.py:48] [95000] global_step=95000, grad_norm=0.5, loss=1.39243 +I0916 08:45:40.938431 139691287286976 submission.py:307] 95000) loss = 1.392, grad_norm = 0.500 +I0916 08:50:20.290577 139670943880960 logging_writer.py:48] [95500] global_step=95500, grad_norm=0.5, loss=1.28976 +I0916 08:50:20.298031 139691287286976 submission.py:307] 95500) loss = 1.290, grad_norm = 0.500 +I0916 08:51:29.382688 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 08:52:51.054981 139691287286976 spec.py:346] Evaluating on the validation split. +I0916 08:53:56.521329 139691287286976 spec.py:363] Evaluating on the test split. +I0916 08:53:57.437096 139691287286976 submission_runner.py:516] Time since start: 82175.39s, Step: 95603, {'train/accuracy': 0.75619140625, 'train/loss': 2.0707742309570314, 'validation/accuracy': 0.62796, 'validation/loss': 2.645846875, 'validation/num_examples': 50000, 'test/accuracy': 0.6269, 'test/loss': 1.6623095703125, 'test/num_examples': 10000, 'score': 77210.85782265663, 'total_duration': 82175.38922595978, 'accumulated_submission_time': 77210.85782265663, 'accumulated_eval_time': 4713.230769872665, 'accumulated_logging_time': 11.575173377990723} +I0916 08:53:57.696777 139670985844480 logging_writer.py:48] [95603] accumulated_eval_time=4713.23, accumulated_logging_time=11.5752, accumulated_submission_time=77210.9, global_step=95603, preemption_count=0, score=77210.9, test/accuracy=0.6269, test/loss=1.66231, test/num_examples=10000, total_duration=82175.4, train/accuracy=0.756191, train/loss=2.07077, validation/accuracy=0.62796, validation/loss=2.64585, validation/num_examples=50000 +I0916 08:59:47.022126 139671027808000 logging_writer.py:48] [96000] global_step=96000, grad_norm=0.5, loss=3.18451 +I0916 08:59:47.027368 139691287286976 submission.py:307] 96000) loss = 3.185, grad_norm = 0.500 +I0916 09:07:11.120507 139670985844480 logging_writer.py:48] [96500] global_step=96500, grad_norm=0.5, loss=1.33309 +I0916 09:07:11.152031 139691287286976 submission.py:307] 96500) loss = 1.333, grad_norm = 0.500 +I0916 09:12:16.928305 139671027808000 logging_writer.py:48] [97000] global_step=97000, grad_norm=0.5, loss=1.29954 +I0916 09:12:16.932629 139691287286976 submission.py:307] 97000) loss = 1.300, grad_norm = 0.500 +I0916 09:21:01.686863 139670985844480 logging_writer.py:48] [97500] global_step=97500, grad_norm=0.5, loss=3.71801 +I0916 09:21:01.690746 139691287286976 submission.py:307] 97500) loss = 3.718, grad_norm = 0.500 +I0916 09:25:45.897105 139671027808000 logging_writer.py:48] [98000] global_step=98000, grad_norm=0.5, loss=3.49074 +I0916 09:25:45.924355 139691287286976 submission.py:307] 98000) loss = 3.491, grad_norm = 0.500 +I0916 09:32:26.168251 139670985844480 logging_writer.py:48] [98500] global_step=98500, grad_norm=0.5, loss=1.17307 +I0916 09:32:26.173539 139691287286976 submission.py:307] 98500) loss = 1.173, grad_norm = 0.500 +I0916 09:36:53.678458 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 09:38:31.302917 139691287286976 spec.py:346] Evaluating on the validation split. +I0916 09:39:36.965894 139691287286976 spec.py:363] Evaluating on the test split. +I0916 09:39:37.939960 139691287286976 submission_runner.py:516] Time since start: 84915.89s, Step: 98746, {'train/accuracy': 0.7453125, 'train/loss': 2.3645455932617185, 'validation/accuracy': 0.62078, 'validation/loss': 2.8875203125, 'validation/num_examples': 50000, 'test/accuracy': 0.6306, 'test/loss': 1.6648083984375, 'test/num_examples': 10000, 'score': 79778.46589374542, 'total_duration': 84915.89196419716, 'accumulated_submission_time': 79778.46589374542, 'accumulated_eval_time': 4877.492128610611, 'accumulated_logging_time': 11.848497152328491} +I0916 09:39:38.312170 139671027808000 logging_writer.py:48] [98746] accumulated_eval_time=4877.49, accumulated_logging_time=11.8485, accumulated_submission_time=79778.5, global_step=98746, preemption_count=0, score=79778.5, test/accuracy=0.6306, test/loss=1.66481, test/num_examples=10000, total_duration=84915.9, train/accuracy=0.745313, train/loss=2.36455, validation/accuracy=0.62078, validation/loss=2.88752, validation/num_examples=50000 +I0916 09:42:26.805011 139670943880960 logging_writer.py:48] [99000] global_step=99000, grad_norm=0.5, loss=3.36794 +I0916 09:42:26.809157 139691287286976 submission.py:307] 99000) loss = 3.368, grad_norm = 0.500 +I0916 09:49:12.824407 139671027808000 logging_writer.py:48] [99500] global_step=99500, grad_norm=0.5, loss=1.19055 +I0916 09:49:12.829579 139691287286976 submission.py:307] 99500) loss = 1.191, grad_norm = 0.500 +I0916 09:58:02.665905 139670943880960 logging_writer.py:48] [100000] global_step=100000, grad_norm=0.5, loss=1.51627 +I0916 09:58:02.670653 139691287286976 submission.py:307] 100000) loss = 1.516, grad_norm = 0.500 +I0916 10:02:38.813004 139671027808000 logging_writer.py:48] [100500] global_step=100500, grad_norm=0.5, loss=1.1451 +I0916 10:02:38.829644 139691287286976 submission.py:307] 100500) loss = 1.145, grad_norm = 0.500 +I0916 10:09:44.948878 139670943880960 logging_writer.py:48] [101000] global_step=101000, grad_norm=0.5, loss=1.32048 +I0916 10:09:44.952822 139691287286976 submission.py:307] 101000) loss = 1.320, grad_norm = 0.500 +I0916 10:17:15.891955 139671027808000 logging_writer.py:48] [101500] global_step=101500, grad_norm=0.5, loss=1.33079 +I0916 10:17:15.898156 139691287286976 submission.py:307] 101500) loss = 1.331, grad_norm = 0.500 +I0916 10:22:13.261072 139670943880960 logging_writer.py:48] [102000] global_step=102000, grad_norm=0.5, loss=1.35072 +I0916 10:22:13.266932 139691287286976 submission.py:307] 102000) loss = 1.351, grad_norm = 0.500 +I0916 10:22:33.691247 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 10:23:54.937133 139691287286976 spec.py:346] Evaluating on the validation split. +I0916 10:25:01.258544 139691287286976 spec.py:363] Evaluating on the test split. +I0916 10:25:02.196806 139691287286976 submission_runner.py:516] Time since start: 87640.15s, Step: 102025, {'train/accuracy': 0.72435546875, 'train/loss': 2.6406915283203123, 'validation/accuracy': 0.62874, 'validation/loss': 2.642935, 'validation/num_examples': 50000, 'test/accuracy': 0.6328, 'test/loss': 1.66403203125, 'test/num_examples': 10000, 'score': 82344.75841236115, 'total_duration': 87640.1489944458, 'accumulated_submission_time': 82344.75841236115, 'accumulated_eval_time': 5025.998027563095, 'accumulated_logging_time': 12.233155250549316} +I0916 10:25:02.497093 139671044593408 logging_writer.py:48] [102025] accumulated_eval_time=5026, accumulated_logging_time=12.2332, accumulated_submission_time=82344.8, global_step=102025, preemption_count=0, score=82344.8, test/accuracy=0.6328, test/loss=1.66403, test/num_examples=10000, total_duration=87640.1, train/accuracy=0.724355, train/loss=2.64069, validation/accuracy=0.62874, validation/loss=2.64294, validation/num_examples=50000 +I0916 10:32:35.594543 139670985844480 logging_writer.py:48] [102500] global_step=102500, grad_norm=0.5, loss=1.48576 +I0916 10:32:35.600284 139691287286976 submission.py:307] 102500) loss = 1.486, grad_norm = 0.500 +I0916 10:37:52.163444 139671044593408 logging_writer.py:48] [103000] global_step=103000, grad_norm=0.5, loss=1.0711 +I0916 10:37:52.185947 139691287286976 submission.py:307] 103000) loss = 1.071, grad_norm = 0.500 +I0916 10:44:43.461850 139670985844480 logging_writer.py:48] [103500] global_step=103500, grad_norm=0.5, loss=2.57317 +I0916 10:44:43.465839 139691287286976 submission.py:307] 103500) loss = 2.573, grad_norm = 0.500 +I0916 10:52:04.337558 139671044593408 logging_writer.py:48] [104000] global_step=104000, grad_norm=0.5, loss=1.02784 +I0916 10:52:04.349837 139691287286976 submission.py:307] 104000) loss = 1.028, grad_norm = 0.500 +I0916 10:57:08.215361 139670985844480 logging_writer.py:48] [104500] global_step=104500, grad_norm=0.5, loss=1.18605 +I0916 10:57:08.219210 139691287286976 submission.py:307] 104500) loss = 1.186, grad_norm = 0.500 +I0916 11:05:31.735126 139671044593408 logging_writer.py:48] [105000] global_step=105000, grad_norm=0.5, loss=1.18215 +I0916 11:05:31.739180 139691287286976 submission.py:307] 105000) loss = 1.182, grad_norm = 0.500 +I0916 11:07:57.736461 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 11:09:15.997373 139691287286976 spec.py:346] Evaluating on the validation split. +I0916 11:10:21.397859 139691287286976 spec.py:363] Evaluating on the test split. +I0916 11:10:22.308821 139691287286976 submission_runner.py:516] Time since start: 90360.26s, Step: 105239, {'train/accuracy': 0.77421875, 'train/loss': 2.0628004455566407, 'validation/accuracy': 0.66012, 'validation/loss': 2.18563234375, 'validation/num_examples': 50000, 'test/accuracy': 0.6329, 'test/loss': 1.6799451171875, 'test/num_examples': 10000, 'score': 84911.81944584846, 'total_duration': 90360.26087784767, 'accumulated_submission_time': 84911.81944584846, 'accumulated_eval_time': 5170.570199251175, 'accumulated_logging_time': 12.54542350769043} +I0916 11:10:22.610167 139670943880960 logging_writer.py:48] [105239] accumulated_eval_time=5170.57, accumulated_logging_time=12.5454, accumulated_submission_time=84911.8, global_step=105239, preemption_count=0, score=84911.8, test/accuracy=0.6329, test/loss=1.67995, test/num_examples=10000, total_duration=90360.3, train/accuracy=0.774219, train/loss=2.0628, validation/accuracy=0.66012, validation/loss=2.18563, validation/num_examples=50000 +I0916 11:13:31.755752 139670985844480 logging_writer.py:48] [105500] global_step=105500, grad_norm=0.5, loss=2.76312 +I0916 11:13:31.759725 139691287286976 submission.py:307] 105500) loss = 2.763, grad_norm = 0.500 +I0916 11:21:48.076737 139670943880960 logging_writer.py:48] [106000] global_step=106000, grad_norm=0.5, loss=1.85863 +I0916 11:21:48.082008 139691287286976 submission.py:307] 106000) loss = 1.859, grad_norm = 0.500 +I0916 11:29:18.625185 139670985844480 logging_writer.py:48] [106500] global_step=106500, grad_norm=0.5, loss=1.05023 +I0916 11:29:18.632475 139691287286976 submission.py:307] 106500) loss = 1.050, grad_norm = 0.500 +I0916 11:34:19.427856 139670943880960 logging_writer.py:48] [107000] global_step=107000, grad_norm=0.5, loss=2.94704 +I0916 11:34:19.432162 139691287286976 submission.py:307] 107000) loss = 2.947, grad_norm = 0.500 +I0916 11:43:17.247607 139670985844480 logging_writer.py:48] [107500] global_step=107500, grad_norm=0.5, loss=1.27414 +I0916 11:43:17.251982 139691287286976 submission.py:307] 107500) loss = 1.274, grad_norm = 0.500 +I0916 11:48:07.056993 139670943880960 logging_writer.py:48] [108000] global_step=108000, grad_norm=0.5, loss=2.72966 +I0916 11:48:07.062402 139691287286976 submission.py:307] 108000) loss = 2.730, grad_norm = 0.500 +I0916 11:53:19.050882 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 11:54:35.488428 139691287286976 spec.py:346] Evaluating on the validation split. +I0916 11:55:41.262425 139691287286976 spec.py:363] Evaluating on the test split. +I0916 11:55:42.170752 139691287286976 submission_runner.py:516] Time since start: 93080.12s, Step: 108407, {'train/accuracy': 0.709609375, 'train/loss': 3.0620135498046874, 'validation/accuracy': 0.6479, 'validation/loss': 2.51108796875, 'validation/num_examples': 50000, 'test/accuracy': 0.635, 'test/loss': 1.6590978515625, 'test/num_examples': 10000, 'score': 87478.88587450981, 'total_duration': 93080.1224527359, 'accumulated_submission_time': 87478.88587450981, 'accumulated_eval_time': 5313.689696073532, 'accumulated_logging_time': 12.857575178146362} +I0916 11:55:42.582622 139670952273664 logging_writer.py:48] [108407] accumulated_eval_time=5313.69, accumulated_logging_time=12.8576, accumulated_submission_time=87478.9, global_step=108407, preemption_count=0, score=87478.9, test/accuracy=0.635, test/loss=1.6591, test/num_examples=10000, total_duration=93080.1, train/accuracy=0.709609, train/loss=3.06201, validation/accuracy=0.6479, validation/loss=2.51109, validation/num_examples=50000 +I0916 11:56:10.913690 139671052986112 logging_writer.py:48] [108500] global_step=108500, grad_norm=0.5, loss=3.19911 +I0916 11:56:10.917013 139691287286976 submission.py:307] 108500) loss = 3.199, grad_norm = 0.500 +I0916 12:04:00.871245 139670952273664 logging_writer.py:48] [109000] global_step=109000, grad_norm=0.5, loss=1.60778 +I0916 12:04:00.876558 139691287286976 submission.py:307] 109000) loss = 1.608, grad_norm = 0.500 +I0916 12:09:33.642735 139671052986112 logging_writer.py:48] [109500] global_step=109500, grad_norm=0.5, loss=1.35354 +I0916 12:09:33.649121 139691287286976 submission.py:307] 109500) loss = 1.354, grad_norm = 0.500 +I0916 12:18:03.138456 139670952273664 logging_writer.py:48] [110000] global_step=110000, grad_norm=0.5, loss=2.08911 +I0916 12:18:03.142538 139691287286976 submission.py:307] 110000) loss = 2.089, grad_norm = 0.500 +I0916 12:22:46.374302 139671052986112 logging_writer.py:48] [110500] global_step=110500, grad_norm=0.5, loss=1.07855 +I0916 12:22:46.380111 139691287286976 submission.py:307] 110500) loss = 1.079, grad_norm = 0.500 +I0916 12:29:31.938764 139670952273664 logging_writer.py:48] [111000] global_step=111000, grad_norm=0.5, loss=1.25318 +I0916 12:29:31.943631 139691287286976 submission.py:307] 111000) loss = 1.253, grad_norm = 0.500 +I0916 12:36:46.276201 139671052986112 logging_writer.py:48] [111500] global_step=111500, grad_norm=0.5, loss=3.12183 +I0916 12:36:46.289133 139691287286976 submission.py:307] 111500) loss = 3.122, grad_norm = 0.500 +I0916 12:38:38.045320 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 12:39:53.820591 139691287286976 spec.py:346] Evaluating on the validation split. +I0916 12:40:59.714139 139691287286976 spec.py:363] Evaluating on the test split. +I0916 12:41:00.626636 139691287286976 submission_runner.py:516] Time since start: 95798.58s, Step: 111719, {'train/accuracy': 0.74818359375, 'train/loss': 2.5472946166992188, 'validation/accuracy': 0.6397, 'validation/loss': 2.67240375, 'validation/num_examples': 50000, 'test/accuracy': 0.6391, 'test/loss': 1.663360546875, 'test/num_examples': 10000, 'score': 90045.97619056702, 'total_duration': 95798.57874584198, 'accumulated_submission_time': 90045.97619056702, 'accumulated_eval_time': 5456.271093606949, 'accumulated_logging_time': 13.28238844871521} +I0916 12:41:01.011144 139671027808000 logging_writer.py:48] [111719] accumulated_eval_time=5456.27, accumulated_logging_time=13.2824, accumulated_submission_time=90046, global_step=111719, preemption_count=0, score=90046, test/accuracy=0.6391, test/loss=1.66336, test/num_examples=10000, total_duration=95798.6, train/accuracy=0.748184, train/loss=2.54729, validation/accuracy=0.6397, validation/loss=2.6724, validation/num_examples=50000 +I0916 12:44:48.307396 139671925417728 logging_writer.py:48] [112000] global_step=112000, grad_norm=0.5, loss=1.28334 +I0916 12:44:48.312818 139691287286976 submission.py:307] 112000) loss = 1.283, grad_norm = 0.500 +I0916 12:53:50.780731 139671027808000 logging_writer.py:48] [112500] global_step=112500, grad_norm=0.5, loss=1.82526 +I0916 12:53:50.784701 139691287286976 submission.py:307] 112500) loss = 1.825, grad_norm = 0.500 +I0916 12:58:56.958751 139671925417728 logging_writer.py:48] [113000] global_step=113000, grad_norm=0.5, loss=1.73489 +I0916 12:58:56.964152 139691287286976 submission.py:307] 113000) loss = 1.735, grad_norm = 0.500 +I0916 13:05:48.687820 139671027808000 logging_writer.py:48] [113500] global_step=113500, grad_norm=0.5, loss=2.97629 +I0916 13:05:48.693568 139691287286976 submission.py:307] 113500) loss = 2.976, grad_norm = 0.500 +I0916 13:13:13.864212 139671925417728 logging_writer.py:48] [114000] global_step=114000, grad_norm=0.5, loss=1.8207 +I0916 13:13:13.868923 139691287286976 submission.py:307] 114000) loss = 1.821, grad_norm = 0.500 +I0916 13:18:06.898466 139671027808000 logging_writer.py:48] [114500] global_step=114500, grad_norm=0.5, loss=1.08481 +I0916 13:18:06.902559 139691287286976 submission.py:307] 114500) loss = 1.085, grad_norm = 0.500 +I0916 13:23:58.895156 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 13:25:33.787431 139691287286976 spec.py:346] Evaluating on the validation split. +I0916 13:26:39.417064 139691287286976 spec.py:363] Evaluating on the test split. +I0916 13:26:40.324271 139691287286976 submission_runner.py:516] Time since start: 98538.28s, Step: 114862, {'train/accuracy': 0.76982421875, 'train/loss': 2.211178894042969, 'validation/accuracy': 0.63362, 'validation/loss': 2.6727815625, 'validation/num_examples': 50000, 'test/accuracy': 0.6416, 'test/loss': 1.6446095703125, 'test/num_examples': 10000, 'score': 92615.26302433014, 'total_duration': 98538.27653622627, 'accumulated_submission_time': 92615.26302433014, 'accumulated_eval_time': 5617.700368642807, 'accumulated_logging_time': 13.676774501800537} +I0916 13:26:40.626451 139671044593408 logging_writer.py:48] [114862] accumulated_eval_time=5617.7, accumulated_logging_time=13.6768, accumulated_submission_time=92615.3, global_step=114862, preemption_count=0, score=92615.3, test/accuracy=0.6416, test/loss=1.64461, test/num_examples=10000, total_duration=98538.3, train/accuracy=0.769824, train/loss=2.21118, validation/accuracy=0.63362, validation/loss=2.67278, validation/num_examples=50000 +I0916 13:27:59.357610 139671036200704 logging_writer.py:48] [115000] global_step=115000, grad_norm=0.5, loss=1.32697 +I0916 13:27:59.361025 139691287286976 submission.py:307] 115000) loss = 1.327, grad_norm = 0.500 +I0916 13:34:17.217208 139671044593408 logging_writer.py:48] [115500] global_step=115500, grad_norm=0.5, loss=2.40726 +I0916 13:34:17.221143 139691287286976 submission.py:307] 115500) loss = 2.407, grad_norm = 0.500 +I0916 13:41:33.867410 139671036200704 logging_writer.py:48] [116000] global_step=116000, grad_norm=0.5, loss=1.16769 +I0916 13:41:33.872336 139691287286976 submission.py:307] 116000) loss = 1.168, grad_norm = 0.500 +I0916 13:48:36.245762 139671044593408 logging_writer.py:48] [116500] global_step=116500, grad_norm=0.5, loss=1.08458 +I0916 13:48:36.260087 139691287286976 submission.py:307] 116500) loss = 1.085, grad_norm = 0.500 +I0916 13:53:35.264772 139671036200704 logging_writer.py:48] [117000] global_step=117000, grad_norm=0.5, loss=2.14756 +I0916 13:53:35.268873 139691287286976 submission.py:307] 117000) loss = 2.148, grad_norm = 0.500 +I0916 14:02:12.507729 139671044593408 logging_writer.py:48] [117500] global_step=117500, grad_norm=0.5, loss=2.06875 +I0916 14:02:12.513950 139691287286976 submission.py:307] 117500) loss = 2.069, grad_norm = 0.500 +I0916 14:07:14.070384 139671036200704 logging_writer.py:48] [118000] global_step=118000, grad_norm=0.5, loss=1.15248 +I0916 14:07:14.075945 139691287286976 submission.py:307] 118000) loss = 1.152, grad_norm = 0.500 +I0916 14:09:42.612003 139691287286976 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 14:10:57.444152 139691287286976 spec.py:346] Evaluating on the validation split. +I0916 14:12:03.121076 139691287286976 spec.py:363] Evaluating on the test split. +I0916 14:12:04.043033 139691287286976 submission_runner.py:516] Time since start: 101262.00s, Step: 118220, {'train/accuracy': 0.7598046875, 'train/loss': 2.4921820068359377, 'validation/accuracy': 0.64602, 'validation/loss': 2.53122203125, 'validation/num_examples': 50000, 'test/accuracy': 0.641, 'test/loss': 1.67543125, 'test/num_examples': 10000, 'score': 95188.4490017891, 'total_duration': 101261.9952890873, 'accumulated_submission_time': 95188.4490017891, 'accumulated_eval_time': 5759.131661653519, 'accumulated_logging_time': 13.992690801620483} +I0916 14:12:04.355682 139671002629888 logging_writer.py:48] [118220] accumulated_eval_time=5759.13, accumulated_logging_time=13.9927, accumulated_submission_time=95188.4, global_step=118220, preemption_count=0, score=95188.4, test/accuracy=0.641, test/loss=1.67543, test/num_examples=10000, total_duration=101262, train/accuracy=0.759805, train/loss=2.49218, validation/accuracy=0.64602, validation/loss=2.53122, validation/num_examples=50000 +I0916 14:16:02.528849 139671078164224 logging_writer.py:48] [118500] global_step=118500, grad_norm=0.5, loss=1.8225 +I0916 14:16:02.532813 139691287286976 submission.py:307] 118500) loss = 1.822, grad_norm = 0.500 +I0916 14:23:34.979058 139671002629888 logging_writer.py:48] [119000] global_step=119000, grad_norm=0.5, loss=1.30486 +I0916 14:23:34.983060 139691287286976 submission.py:307] 119000) loss = 1.305, grad_norm = 0.500 +I0916 14:28:42.749705 139671078164224 logging_writer.py:48] [119500] global_step=119500, grad_norm=0.5, loss=1.25355 +I0916 14:28:42.756515 139691287286976 submission.py:307] 119500) loss = 1.254, grad_norm = 0.500 +I0916 14:37:22.785238 139671002629888 logging_writer.py:48] [120000] global_step=120000, grad_norm=0.5, loss=2.58695 +I0916 14:37:22.789227 139691287286976 submission.py:307] 120000) loss = 2.587, grad_norm = 0.500 +I0916 14:42:20.563073 139671078164224 logging_writer.py:48] [120500] global_step=120500, grad_norm=0.5, loss=1.18074 +I0916 14:42:20.570510 139691287286976 submission.py:307] 120500) loss = 1.181, grad_norm = 0.500 +I0916 14:48:54.134943 139671002629888 logging_writer.py:48] [121000] global_step=121000, grad_norm=0.5, loss=2.7899 +I0916 14:48:54.142232 139691287286976 submission.py:307] 121000) loss = 2.790, grad_norm = 0.500 +I0916 14:54:58.038960 139671078164224 logging_writer.py:48] [121348] global_step=121348, preemption_count=0, score=97758.8 +I0916 14:55:05.983500 139691287286976 submission_runner.py:857] Final imagenet_vit score: 97758.78199744225 diff --git a/logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..a18bf720 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,39 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/accuracy,test/loss,test/num_examples,total_duration,train/accuracy,train/loss,validation/accuracy,validation/loss,validation/num_examples +266.90091943740845,0.0,121.32478308677672,1,0,121.32478308677672,0.0013,6.90775546875,10000,388.9585015773773,0.00228515625,6.90775634765625,0.00176,6.90775625,50000 +433.4192156791687,0.2330889701843261,2692.0896463394165,2371,0,2692.0896463394165,0.0486,5.6401859375,10000,3129.844084262848,0.055859375,5.700372314453125,0.0493,5.87810625,50000 +583.4502947330475,0.3073396682739258,5262.350856781006,5610,0,5262.350856781006,0.1741,4.29425546875,10000,5854.4937126636505,0.189765625,4.9383306884765625,0.17356,4.7599465625,50000 +725.8568375110626,0.4261224269866943,7834.110359668732,8777,0,7834.110359668732,0.2762,3.52323125,10000,8572.682470083237,0.324140625,4.036053466796875,0.287,4.09793375,50000 +865.246161699295,0.5769710540771484,10411.95853638649,12004,0,10411.95853638649,0.3526,3.036040234375,10000,11298.288755893707,0.40556640625,3.770957641601562,0.36128,3.826253125,50000 +1008.971207857132,0.8551476001739502,12979.315444469452,15268,0,12979.315444469452,0.4116,2.7012171875,10000,14017.990199327469,0.479921875,3.2422940063476564,0.41712,3.508525,50000 +1166.0138342380524,1.1592702865600586,15546.428710222244,18417,0,15546.428710222244,0.4518,2.47650390625,10000,16751.14104294777,0.54462890625,2.800125732421875,0.46806,3.2255234375,50000 +1312.8532564640043,2.128077268600464,18114.06486916542,21768,0,18114.06486916542,0.4811,2.323434375,10000,19475.07040452957,0.54513671875,3.107976379394531,0.49176,3.171916875,50000 +1471.6794302463531,2.4110121726989746,20681.5164604187,24895,0,20681.5164604187,0.502,2.215092578125,10000,22209.77625346184,0.56482421875,3.170403137207032,0.5066,3.2526559375,50000 +1617.216381072998,3.4876534938812256,23248.10187292099,28226,0,23248.10187292099,0.5237,2.1259302734375,10000,24931.695034742355,0.5911328125,2.883991394042969,0.5363,2.76262,50000 +1756.5567178726196,4.614113569259644,25816.279580831528,31455,0,25816.279580831528,0.5334,2.0450654296875,10000,27648.096289157867,0.663359375,2.111064910888672,0.5554,2.83841125,50000 +1908.323546409607,4.924053430557251,28384.168689012527,34457,0,28384.168689012527,0.5457,2.0043236328125,10000,30376.761737585068,0.625390625,2.8165191650390624,0.5694,2.614239375,50000 +2051.183312892914,5.2484130859375,30951.68151497841,37631,0,30951.68151497841,0.5491,1.96443515625,10000,33095.715420007706,0.65158203125,2.6507022094726564,0.57466,2.8467275,50000 +2199.3028275966644,5.618121862411499,33523.692170619965,40793,0,33523.692170619965,0.5644,1.912332421875,10000,35824.94554591179,0.66376953125,2.469996032714844,0.56806,2.9299578125,50000 +2339.386500597,5.942543029785156,36090.76889109612,44048,0,36090.76889109612,0.5681,1.8966951171875,10000,38540.7880191803,0.66783203125,2.445870971679688,0.5822,2.6944565625,50000 +2490.3515722751617,6.298062562942505,38661.46665644646,47187,0,38661.46665644646,0.5785,1.8651451171875,10000,41271.61074280739,0.61841796875,3.3203265380859377,0.58706,2.7905684375,50000 +2640.9342324733734,6.681631803512573,41228.18127465248,50538,0,41228.18127465248,0.5798,1.82652265625,10000,43997.95974302292,0.663828125,2.6336328125,0.59092,2.7246275,50000 +2798.588620901108,7.0068535804748535,43798.88992142677,53663,0,43798.88992142677,0.587,1.7989046875,10000,46735.01598310471,0.66783203125,2.716911010742188,0.6055,2.58237921875,50000 +2947.053974866867,7.303242683410644,46373.18920493126,56977,0,46373.18920493126,0.596,1.776692578125,10000,49466.45277762413,0.67353515625,2.700517578125,0.6015,2.7015803125,50000 +3085.3943016529083,7.672607660293579,48942.8839867115,60128,0,48942.8839867115,0.5981,1.76907109375,10000,52183.45779824257,0.71884765625,2.027147674560547,0.6182,2.40730015625,50000 +3236.477827072144,8.006976842880249,51509.77036523819,63303,0,51509.77036523819,0.6045,1.758034765625,10000,54910.55043578148,0.68916015625,2.5399473571777342,0.6227,2.3859615625,50000 +3377.9268865585327,8.292526960372925,54081.4677426815,66544,0,54081.4677426815,0.6037,1.736313671875,10000,57632.322525024414,0.67556640625,2.814586181640625,0.61188,2.5821090625,50000 +3528.137953519821,8.600436449050903,56649.6544585228,69665,0,56649.6544585228,0.6067,1.7296421875,10000,60359.17927026749,0.69140625,2.551528015136719,0.61948,2.4735353125,50000 +3671.800485849381,8.961181879043579,59218.12683725357,73022,0,59218.12683725357,0.6071,1.7324955078125,10000,63080.01647400856,0.713828125,2.382465362548828,0.6335,2.30428890625,50000 +3831.300248384476,9.277946949005129,61793.17218565941,76160,0,61793.17218565941,0.6167,1.67845546875,10000,65822.94068861008,0.72046875,2.278394012451172,0.63344,2.3765715625,50000 +3983.537580013275,9.606787919998167,64361.67498803139,79490,0,64361.67498803139,0.6165,1.7130814453125,10000,68552.48768758774,0.68509765625,2.83866455078125,0.61306,2.6923934375,50000 +4120.435905694962,9.923115968704224,66933.0193309784,82666,0,66933.0193309784,0.6212,1.6881970703125,10000,71269.16496658325,0.73498046875,2.2339772033691405,0.63742,2.43264578125,50000 +4266.311295747757,10.376159906387327,69504.26203966141,85838,0,69504.26203966141,0.6217,1.6680759765625,10000,73995.53771471977,0.7084765625,2.568540344238281,0.61908,2.64155625,50000 +4407.885825872421,10.789963960647585,72072.62209844589,89082,0,72072.62209844589,0.6228,1.673147265625,10000,76714.18280935287,0.73923828125,2.213905029296875,0.62456,2.56997015625,50000 +4565.176446676254,11.278878450393677,74643.82851338387,92224,0,74643.82851338387,0.6227,1.670181640625,10000,79451.50010681152,0.72103515625,2.4955255126953126,0.63528,2.4236353125,50000 +4713.230769872665,11.575173377990724,77210.85782265663,95603,0,77210.85782265663,0.6269,1.6623095703125,10000,82175.38922595978,0.75619140625,2.0707742309570314,0.62796,2.645846875,50000 +4877.492128610611,11.848497152328491,79778.46589374542,98746,0,79778.46589374542,0.6306,1.6648083984375,10000,84915.89196419716,0.7453125,2.3645455932617185,0.62078,2.8875203125,50000 +5025.998027563095,12.233155250549316,82344.75841236115,102025,0,82344.75841236115,0.6328,1.66403203125,10000,87640.1489944458,0.72435546875,2.6406915283203123,0.62874,2.642935,50000 +5170.570199251175,12.54542350769043,84911.81944584846,105239,0,84911.81944584846,0.6329,1.6799451171875,10000,90360.26087784769,0.77421875,2.0628004455566407,0.66012,2.18563234375,50000 +5313.689696073532,12.857575178146362,87478.88587450981,108407,0,87478.88587450981,0.635,1.6590978515625,10000,93080.1224527359,0.709609375,3.0620135498046874,0.6479,2.51108796875,50000 +5456.271093606949,13.28238844871521,90045.97619056702,111719,0,90045.97619056702,0.6391,1.663360546875,10000,95798.57874584198,0.74818359375,2.5472946166992188,0.6397,2.67240375,50000 +5617.700368642807,13.676774501800535,92615.26302433014,114862,0,92615.26302433014,0.6416,1.6446095703125,10000,98538.27653622629,0.76982421875,2.211178894042969,0.63362,2.6727815625,50000 +5759.131661653519,13.992690801620483,95188.4490017891,118220,0,95188.4490017891,0.641,1.67543125,10000,101261.9952890873,0.7598046875,2.4921820068359377,0.64602,2.53122203125,50000 diff --git a/logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/trial_1/events.out.tfevents.1789466662.f86f86246ea5.38.0.v2 b/logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/trial_1/events.out.tfevents.1789466662.f86f86246ea5.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..c5f51a53e500211c23308a0109eaf5303681f53a GIT binary patch literal 112693 zcmbT)3Lx zhus2Kosr%Su3iK6C#uV;c{#Xxx_QiZaa*cm@8;p4v&3tGi;hF>fA^fT)MJ5{gU8y8 zswnNhX_aTvN6M@-de);hO{=SH7g3BbMD70>BGmNka*X~)gfCxhEwg%GrgQD_%Qe#K zCEHWBl~?UK(Y3R;U*chBzx=rJgUrI3r7t58zgZu>6Faxr%aH6tiEh}BFJ^gqB#!B>=wDZ zI55*LVP$F#N9HwWMDPB81B*-UPM5KG$pmW2YRb0sa`5!xoblv&Q3x~+jJyn!zrT!$ z5v(6U3`yfGz>OiN%#HsZNyl!{vt*i9GZ8wn+OqP32r9~?vl*biZm_tj`!{OQY-KB% zKh5s{w~o|gRb`uc)_#EgTRvkVKM#lDgKK7fhGn?4H8Whe4Bfo`zm(_we-YMB|9?3# zE0RnhCw=|M?2&&D<3o{Cj7*F&lVc>?PqtI-LSD4Q#m>uN-W<;*a~Cb}^!$I`9Ipk7 z_z%v{FCu>E-_QvT-gYwYG?-9*Sv}cyWa$6@y9>lk@Z5mMc;lKux8Z-|mVa@Lk})=7 z;tXZ`$|{j@E^dyF3tSx`v34w|Wio~zHT;|S^w#Vmnd>G@BL7a{I}}1XUo87E@NY<) zJ?~%3+_hptddv2bm2>xSa9HGCd-~_tyDf3``fn0Ahtp*A=8rmmlWyI!{vlIVsj=|y zo>Kp=c(3e`Gpy@TznRhWebF#E9{0sAvyJY;`FP)~$A1SP_W5|a!^Qy=y>9jshBW5x z?UUJinZgQJcTjw~!;GdY+Q+7#V%zzIji|WKMDQyl@EYU%V)R{5Sk;UYDw=Q=6WEHr zA)yCA(M)#qf1~M&HXm-V6t5<^fI_|#6Tz=AL4}J}FMm*YCs+%QLz1iD%!Q);>JgxL zeU}+cSA-MT%XJ!T^K6~3sLQDAZv6q^(sg^H$J#Z>lO$OSZ71`0zBW;9(< z>dpFG9Gmnz4HQX+Oa#9o3>7V&%svZ>xnbTyg%np|%2tfAzi1AMv8R~PbVZ_4GS0=} zi~Y<&ar6Qc!LLX_#jyMf+dxruZM{&@jH?*OR>bdFeE}4?$;@cFVw2=eRE+n2^92-t z?lBSkiUL$jo|^I*6c>8z5h|p)iV& z!{EwOsefraD7t5a3l%b4#aOmt>YOuLpg8}G8BJF>Ph;&-ecBn_1V!~rCW2ofhl(Xh zlCMFbz52XRA$u@yl9_9~z-*vO2gE4In9t{28qLw+KU!jVMBdM1Lfns-l zqEOLX428)WBVAC;`M`{(D^8EUil2)@=bwi_apWr#!LQIm#dwWqcTmI(&JrqGa1}$? zb8+MJqZ&}WHDgB86~{|hJ8g({r$?Z0uwo+k6(*?2SiRycD9*XQ5Gq=76^d-dvoXa@ zLGf`ZGn%e&yTMYphQ12~g}pBm!LP7F#SvycU{hOiS7f6_cNK%bcJskYmah@ubBaJ z(aVL2;8*ygVn|BPbWoh}>m^jQ;VL+v?qH`8F`(GEjTuc>y#11hpNq^j+FL-;`XCd* zuLwegr>0H`DBi9x7Ah3D3QhKk^2(oZ1Qgx@%xJp8X!>PT6vs@O1Pax?Oa#9o7!~*B zZzh6bs@)i&qAgcZ#9mQf0z3wQ!q}Y|O;@CLW%a18omv`$!g>`G!LJBIg`UKnGEl_p z&lD=!aTS-?ic3vS8G<5X5Hp&tXfuPQsBv1L2nxfIOa#B;1S+JbpPB=TCS}e-g(6q+ zgso`r`2s)Pzp9zhbVcz6*5~5zl?4w$(MM|Ge+R&?h(W~{nLb@Wk)gIi7}B1r@MbGC z))mi!xfs`*8BJHnOgJT%6ML7!Hcu?@R>0A^{bW-K+4gq?yh42^Af<3U#*PRP-!4P_${!jHW9* zGOpnsCHJJ~DNqblWg_?$si@d!H~T2eg<}43p`s&KQ6$T{>a{d2=?@C05@s}AaomRW zclW8%cK1MG@Rf<+S7f0gcKz`0pm5f`EL3#jDh$|j@n(8u1Spb6GNb8=X_u35F8Zv0 zSPF{Q)0qf(8g2E?< ziQrchphAC56CF_G`92XUl(~vCY(-6I&IVAl3}8mn75lcadQ_svVMkEd?qnkP6(y*+ z{(OKnD4u#03Kc3`g(>@Uk+a(pd(^8n%xJpec)=}PQ4$H0@P1VMW+sAPQI3kxUxP-0 zLN4u#P|<~}2w=~}P1jJoljHlC8BJHXSjD0uRPs?C%!N!o6Tz>jM#cK`Q@epeFSN-b z!QU6ET*V9aT&z^zp9(9gTqT>UGZ`_>n~P0)uJ3w^cu@V@GB%yk&$}*4JbPLs|giqT*VC; z*7f4<_>qG^v3v(Jny#qSWc{r-`@yL`pm=(7R{7`l~-;8)0@!bE*0ej}z`Whqp2=PDxDiphIb*MLIi4>Ovs zSgOVPbPups3I;{K^!Wb{fM3xT6?40eNrky6STI`{qQOtwKdlt|F4H7+fJ801AU|%xJpey(a5=G1X2A?}2##VIuey z>Zr&(XOBB=UReU+&c$~4r$D2x}tX@)*e;u_-if9MgAiuf?uJDil15r zAut!e@}h-`UR=ekW~>#pSa&~u$GW(f8BJG|o=w9Qm2BjVo!0Lw6Tz?0Muqc~IZZ$@ zr#w!m(B>*eNwE}X2R_Ab?LEFTqv?vrP4A#0L&Ih<%*C|E*Z(^JeuW+?EL{xtfWoZV zJz+?1uHrp=Md|y!8wD%snF2GKt~j#nEGjBm@3{<$Q))~Ezrp|&&TjG*py>Ghxlp0Q zReWg5nu}{$-i<)wyeCW2ov2o-1hH&q11iy_rQg&tS2iv77T zTQF%FC}O5Dqv;Ao6V^^UV^hspP=weq5&Q}hRCtXa8wd(3C23c|UrBwqioI-wnpdPd zDE6x{qv?uxUDmz94(Z{WLD5a0iQrd^Kt++A+%iy1nyn;M=yMfeO;{`HW6F(vpm^lS zjHW9BRWfl!-AX>J3W^nrnFxNxC{z@CQIE=Z9 zWxrT+p%mETD=6&RGo$GW<$O4k#vdWg_?$E~qe*yATeFOr!Hc#UQRCkFDr> zFf{}ek*3UOx}tC5LsWzecECG1JuI0BeuXFuCMF?}gi4CN|b{9q~W?5!CGieKZH(R9Tc zf7Z7!-i2m(z1U{WMDQzCq2h2Rlm~a(szOfW;K3(u>mYo^QXu9HX-eX)* z%QMn@!dx85Wg_?$zNonPJarZ*J}LheDk#57n_a_FEDjrs_sq9yGNb7VC0*8EtP!8m zRX`zQz(nvX)}bQXcK=sUJScAICip99ICn0#er753c7MfB_vA*c{u@nKBrSP~b0K4X za5pG2J2MgdiuI^CXLfiXC=zBV3y;H;tBCr-QuO&_ehNMp9TzgA>55b-)=oQgS#A?h zb z;8*yg;$_Z0+-civFcvCCauqr371e6y#dJ_~*v5>eD=tSBq9Vd9x+5s^LYWADMF1)m z6zsvLAD32-5h~2Mii?%3xo8us-~oz`N@M>UO;_~Y!MacTB_R!;&`vdBBKQ@7sMw}% zf_K##1y^DFi8U92 z>C>izqHG;Any#>%%W5ExHZH~A!ff2eMDQzuQK8f5gd-@VJz|B5aa_gZ_bkP^P7jBG zLUT1UnyxtBl6Ci>qD#A>Fc&IYmp5o<0oEJA%jv1c7Kny#q4TaJpP z%0r_;@g|Uo;8%p9!gPI_1c~^GoQd!^kO%VE~|FG1H~W$LXOa#B;C@K$L#k6Ovxj0{LjK9}gTFs26EAs#R!4>7KCe;-bJEdR!cL4l~XjG`)?tu-& z>r=Tf#G0!JddgC$oj;TaD=NHE#ebveit-(msE|k-jy>w6JQKmMID-oL#%m>EF1+3U z2#;eLS5f?&rTCSnm=B8b4a{h|Vy^^ik6Icou?ptGX%7>@uZTg#_pe1Jpor2`^APM& z)47VE7c9jcGhMuU;L(Q}O;?0;VBG_G-}BEqP{P>I+FTVBo#qV}DrRyOPqSEx zWlFYfK(T2NGn%gGpjM2E4j$8vfWm1B6Tz>zgo=oV6~jOg`EDC-Q#J^%7xRjl2!6#CR1B!_Yypb%Edzv#*<8iV`z%Ed=_9v5A>WZ1 zO;`AqR-r=Ldlf$O-&UQ8;8$Ej#b~{%Fi`mPIUrQna231QE6T`uUJ58AjhWGOMfIcK zs5rWNG5&pFY053mcp!j@0N~oB_Rm8Jbl-AKsZ9x(LoEc456tJ&)iI?L1 z;Bz7Qnu*|7#G^veQ4PPfA08VgRM>JA!>_VdRH}prcqMSr}Lv+xoV!LLX{ z#qD?>3kVuk%a2@4RaoC-DfT`%y$TdhRhZFqMcAq@sOY(85kA}8y9X1&uegm0??t-! zMzY6R-GqvHT*U~sqNSA?uBa(}nbCB`BM;WKTjI6SRG5q9LzxJEMJg)#{8-!*-m#}L z>2|4LMO9K>=5=)GGkg$ae2CUcctzvuS)$=|M;ZI`-LkXI(|&(n?Ka*z@RN+MgD8U^ z@&n(1clA=I>i{|8dvb$$?Mo+1C~sfNGtWSX*C+2s0O=nn8cv5)u_42j&gun_guS8+ ze#nXu@aCfF^S$K(;*+29M*zvAye{X~vO5TgdeN~QAhU`^!^MQ0@iMyrkgQLl3|@$D z971j-MQH(KT9=C}N7c?q7UlgWO_tw8$kN`bTL9wLQ#70o`NE!&M32A^0GX~Y%HV}a z{6L6*x2oCj+E1BwzuJ$kg(SDB>xHHn>GKA|)b{us*Bd^uGc`p)Ja07hTyP1`%hfLx z`-c4exCq`g8(}C4sqdAh=^sbqA^j;FA^~q`EE_3GtLvSn9v!aZap?`RD}`53W=<3( z{nJZL6;s~Mffsmwog+%*f2-*|Ide4baJke8R?-JoQI23Gab9ccxauP6S|`o781(n7 zr(O=9C}X%%l+nQVnl4@yTB!FoYfn|y92p;fQ5N^brf$PDN5V4tqwl>F*5kI_qVaXS z*>vqeQw-|RG;5XF-vjY5yn0MV@rWo$+*g}&4o`pS9DkF)shaI8_sD#S7NyqzZqt*o z*|@VC$z17a`Zw>~hVEf9^KXdq#Co}D!5LR903-KbwuOBm=dLJ{cVEzJi>EKcS|t}= zb!(I@N@>{ZO>Lg`83TJ|PwAi@f-NS6^3I#1d$MqK8c*0;1}nTwUNoG(SC+H)%KKWk zg5jJPcNAsto)eSKcuq1e2gL#8__>V%0>~B0OGWAr)-M0rf>Y z05W2sD1#Rwu>v7lGBX?jV*Ju0SpeBh`Q3Jwjsr+^XHf<( zq~H`nMr*lFfGcYH<(`QG$opP(HN4UZ!|}orW2%GqjoRH31@SbzOsPY7UfhZf{lAq7 z|0ig8@9&F3>T7tHuJ*x0vheFR0vg`998p?b4R7UyU_36V)epWx!<+Y3l=M#xuebFv z1!#C?pG1lL4R55mJ{pg&y{!N%$@7~iN3fDO4KKiXGwK}o*+@deTi!^ESpyAhc>QkY zmqWvACL_w?HoW^D5x5@bd_MRJ)}yO}XnY+F@5`4<7&LjvR~u+}%F3c3aT^}z@S+{e z519O2pn`_?L_?HXf5Us2Cy#$%E!vf=4h?UDz9>(uh8H&TAg+iiia?vYjjQ~7ovI(AzfXzR|6z&|DOi}NH*nHbt;`V;*4aq30ek_*~dh~ z>5$LtV)V;W@)ST;pAlv7LK1QiQt79m0gyjUGAEC#U0UgspFzbAXF*D4xnBoJXj{>6 zF(H4NE!qVTRSi)FFU0meLT2BLD+9>ygFp5NAPJP;trYHM?cjD(H+KYx;T6$vI)qb< zGWDzO1ElzlD1#Rw@dF_~4vLx}JU`d>Pz#A?^x1ujQS=KJCv=Ma`qvWb%xF0u3lf+j?23;2g@f4#O zJ=WrR8S+fm8;a4iyP}Z#icwU>PCTT&1M|zF7?nL1rPWo8%+&nwxE@>FdCy( zpNi4T0nyu_7{z@QCGr;|Mdz((+`g_!G_0h)HKH8BO5zlw!>jC2H{$E!^-zqiOI=~s zKm&_WOzzqzP>g)qin6%H=yg^lez00^R4{|}sN6*~zK&uP_qi!9?A-gN)1eqG)D{Ja zTZ}k|rzPVW2F0kRpD4BdV)WG85R1`A^-&+77!4RH$`h*?UD3|L714KguN>ImW?P9O zdG`f%89aT)GS6>8F?uyyl+v(b^ex!Y8urQy=LQA}wwM!?AD?{dv;I~$Afc3B3-tPM5FyJ03@-pA zucc@>9m2VKjZD9;gb*cB1~0_DEkYV4smuaM*h%^I<7+qlAj;$2*XGYfh_8J3Vt_c` z6b+|CzOswa!+xem0g`r4l)(#O{{v$0o9?&`kR$6lrwAbHDbM2uC9Ob6z}>_z0O`6@ zG@K6MG`!xtWnw?Tl1Y9s|wRjzWl3vhiqu*!>U< zr$abPODSdRK$wx7Cg#i`@InfDAtbXf9%n=%ZBnQJQc8Jl&rEqVLb{|4-wTkq$D-k4 zLJmIpWe$+w7orSaNK9XZY?vga2s7e)Ipw_oQb2jo%Jk?MgjhLG-UkrZB++m>q>}v^ zaSiF&6(BF}i86R0zQzc-@o3HoxT4OysL^IZ?b0foU01`KAIW-9c#H1A5%A^aphQs+ zPs5vIy%^7ns-~wpG`#S1QAm9a@7(t*c&+;$IPn-Xythw8X>~QcDX+ZnxK8U0(}0F& z{zjDaPYo~F@wzlLyg%HR|EnlRu#z|pZLBo@k6J>E5-qW*cxE@!wn0f)$qeVy2_&OS%=4L$%3dz-1f`&J| zyC_K9hQ~R)?}1x?L&I~{6{XhS@Dj&Q!-jWbqD)h0cp1i`Jh2*{?L`IrM;8xfj5mON z;kTJ6l6PMSV(kmY#!Gdf;jNh{N@-Zb>#LOE0ej`;Rl_z5wwTA1hws9lvp##~mNzqD zh39S&4Ht8-eBATJ3OFZ^4~Q~&&xw^OE~%#XKlTGiaNm;;1&~w+?lWKA{72!(Y~9o* z;Q(NWu(+YSiHGr&67Y(OF zIGuae-hDj*V*f;x!3znRi;yU5%>lWLb%@DeU$U(4kPsc+r(Y%lMCF7igBN1!j1Zf%`Cs5d)Y`n=H31}IXk$tMIrSSqr{*mSd1(ptU5w5dgLrhtE(7AZ?VPWn%y+EFBGHk zOGQclRE!FTmac(fq_9qu$X|>G-C+GdXRCi^4XmVH0iqniO5zlw_yTRzbv-(^5fr0& zdqf!xEJn&YR`Z}3RfLJMxW&jcV+^jxgwO@J9<5G^#@A7dzBZeQL48(C=nuuHd8{Z% z++xHzyzfKXw}4`F@`fn2{$iv#WEB>pa|@MTKru4CCCU@47*)QXfyGGq{}0x}zL1(F zisaoFf?V~n-zzzN;`F&EesE6Od=+Kzo)bL}JSSHimMj6t{uP%TCf9EIzN@(qlUA&cS(oP< z)B$3&UNoEz;rumR`a1b7KyC$!GI${pD-aUxdL#rsq+K)AG6j%I-$q=?qywyNde(kP ze6RA{=frqAg)=A49l~Dq|NHfNamn53G6&ueA-t9nU$pdBY+3}CjJMyEr_@eL$$E-K z>4pz3FT4V-NI={ThHfcfu-e7B81nHwH_`0_f{PS%kdAE#{`x{0(OLN?u zleF98U#-e9L=sO;GBsq)^V~Lvj|~0$(V5`jZ6|Z{GLceWPuhO$6ZWK3he3T{uP==! z;_50&KU^(w)=I~zT!HJ^&^tubKQ$#AEwg>lP$y*)k^EI@@%o6dz3u3VQkgCJL`Vb6Ql?jsPoKYG2mSos$;1>AVcfQK`5o(u z)UL%i16Y{XKM(6A?DcOtg_!n8PH2NtH@vZeb+{0%gnuCrgpxKV)UqbOEms-C+WKa_jya%5qbP?sSx zd3TA|+wl~Z6k7X1jaHB&Vj9+&4rI9-!0y>KW8Ol+j*}8Z**(|p?~lv0uhdR8SnJQ8 z6XWT-C+7;?S8?8GI4}L)5h1+i#Xkto%kVPqNU->>Ir>~+xw4yL>DBlMS_~$gcny{} zTZr*=3+D=5BAeP5EQf-L5MGP>ZnQiPl86V3U58J7tZT0X(ZLjp)ay{Ryo^Z72g`9a zVm#f#d1Bt)EcyUgerORPyq0OfXmNhucoJAb%=(@eSi(XnmZ$4Gp=F%(uG?UFX+?~u zTR8J#aIoolu>Ai8W}<`OwU~sWW!CFY-@tP1OaDrNWp^0Ga=WN2T8=OG#JBJdZDb@G zPq%Q&`4Y=kEx_W~iU{GgXojKX(bg%_VA1jNoja{|jrm7VEKfgnM9Z=6&+zlodJQq2 zZs9z9_(H`-8Z7Y}i4b0kTm)K<51cLwmX6=rCJQVz$0?Qv-}j;=uSx>n(%8ItJJEQ$ zh4Xpwy_NS9EFIesA-tCA<7iP=em4d#ww9*TTTK_tOH`dLuhc6ypy_rZ zN!%95IY1jNwQ6XAJNFW?^|wIFP7&Awrx<+g2`zByAtF$$7PwGf8s}8E*=l3hB}^iT zOx|5WE(#}ORpDe)Xn{JXiI|49z#)zUFT(DbcyjST!H!dYin4qDuw^x;!Gp{fz!iG^ zWnw&i_x!|u03&Y2^h7u>Dv3l0?|CUWh3BQ`Nh`FdAJ~&DuoRx9SOyif!|7=!U-1Ym zy^ay%=@w3-_0JlV36`SML=Q6y68Yfb?%jqsp?Z{LxI<}|P?5m#4pTHnhGKTUJCC8t1h zdaO-E{Zn%q{OQqaXijVT6OsJQsg0imGB>I>!FTXC8%_iXRurc>wM}V>z^?=JRiHTy zvLHel*qrvgncNqe(|BtljN6=~cOAinnQ-oPIxNhYHbg)j&1tLOX-w)b{pL0_Cm$yw zN!;edIl#nQ)6PJ1s_`IV>u*l+RhO|jy}jxp3(aZsY9dgq<|Jjw`h)ZGx4m+(OO*N( znY_Einj}1hyFEtig61@E7ZKC2=5(UkLJoG%ngJ_s3wE3n$&}sGJ(IOVY#%&jC+wbs zM-b!byC>(~hKbqnrEp#5-mDupyLOFLKcrY%%SfTcf0ATVn3vBV ziScv`XI?tazuX@z9yLS=uciDUTEa>LusMCt$$uoUls=_c0ynfr%Y~0K?t-O55iy=_ z;k0up_re)qk^M%5@LIB;qQ(8e>Y=d4JSKH-Yg0Qf1vwOpnr#QP%y9pPcRS=}6XV6S z1TVD2Z-fsz5FxylgdDWkYN?jO#a93ArMUu2QeK@caOLFo*hHjl-{?ULZ1tT;;%R{s zb{XNhaa|KJ8d{*Tn77<(3WT0xeKmo`|cf1*T-`;gQ8J=xGHlP+ygZ z`ll8cGGK=}v_QSyL?nL;G?14<=G*%+_u$h}Y(xYJRurcN4r$H0*K#KIz;9@Qmq!pG z4QzoO7o?wp7T9qD5yowS24S(dFctfie}sj(c@_~+M+=;HhgD&2L z2YB<*07YnlH#~{h`deU`#$9ZIN}bMoKnvXJO9YD50xuiR!A0x#>e4CL@nQpsOx|5$ zeI8Cm=QGQ@K?{5pLc}zz1xl5^o&dY&@jVs?1Ut^-7nI#|OzCVS<@R-OgtdO~FfpFK zdvfjv#Qknv3FoEnNg{;zyqtJ}=Otii_#Uvt-1#Osr}j#a`kG>Cwtp2`dNg@i43^&y zi1Bm_=c<;oX?;Fe404GOUQ5txv_vUSF$ar(tCQ0Omg_|n%LETOw0v9N{sCBIl!)11*B81ms`yMT3*R3MJ(jv2mif!#0i}*;f{P79GdGWlp<^@}F`jPWd|qM} zZ5RTU-R+4GUW?vmv>2{QDS!*qipWqIyV`kqSyg9qDt*usuih2S+ayAB(vKmMc$!m8 zr)D_M-!vOfhUS!dnMkRxIi1&Og$K1{)P*(BoZ{n&xVoB?{|Q4pvO#k!x_4El?REHDe z=@w4g&>4|Z4wmaPi4b0k#1FK{D+TNa%MbHVrMb0Ng34bMi^nb2&RwvnOBz@@P9Vn9 zEu49=-eCP0EaB6M5ME2kFSKOd_S6B(`?H(p2rMOQ8*_h!a3YlTA}kf%%gJCd(hd-f zr&~DpoPOnZz5tfyrbGz8MZz3jR{hT9%nqG$OPTLmyw2@%U54MHYV{|^ z(=D8N87*gn&FSlKB81nHC5aZNX0{1%fl>^(b3tGU+1j|SC-?VvP(@W+xxh4NPG-A_ zB%bD!Gt2<@#R(r8YeREVKS-q1*PQyM$m2m(#Q8=+b832mh^wnPT^Qd3kL<~fHg?dQ z$}SO6|J0n6tt+QObIMF6BKe!spYyEe<#Gm;tbk8P>ntKju%bB4>6w`X0wesNw}j@j zDW3>wU~@_v6l)31DXoME<2I+M3HiAB-G6-F4;JR7Dk7kc=CpA_878T=_jm`*se?qS zXo-v4oHz$~-YD4$nv-99BDVhKG$Y_6HYYPvU0rBSI|mbiVl}7i9jv!Wsduo(-P_TY z$mHE6LZon)7+-FQ?|xglj)-YkbJDrq=?mqU$g(;~M?u_;&vtR_NuEmx#*jy-k;;4@pB`ZZoBute{n zSmr4=!tJB&(wF66xjKayPq%RHZAf3u*9A+O2NA++iIzpnG+!@en3q*o6Uqgaun>x6 z)VXnJ@%;V40xSV{i1Bm_=ibJl@$w;HDSJeO@LIxJpk?a*V^Uywe&N^@huSr^`yjBf(;;LX4+dIBTrS;bHiVux~FSgx9jF4O%MJX5qzl>&Hzq1(rX@>TH2+Gc{0U zbGA)7w7`cyi6ovD*y$MSC+$zSFFXk?aFUe2=y>XDfgd`vUWK%Nf%hV4f%oKyxVl|RHaG4%oGx!Irb?vLfcrkZRk_N3t(wGb6t1G+ABd} zG{urX`x9C$gA)&dC2}4yUQElDisf?^ zYhHS(t+ED7zs6%k%ueoNH0e*`b=0b$nSYcc7JmIcG?mB3PQ_(qh#a`8IFB3B)V+nN5& z>gixuP9D2_J}*`kIrS3+w)w&-(S-p*d++5pi`jC)f4H zIBR1!&DaX}&o)Xt7M-boYECPh2AjhDv(B zjlWZg+ql|J$+OU$4tFL(8rYnAwp-{8&BK zqvxaL!X3FiV7VDgjHg>TS2df8DtueShzKHt*CMBm7D=0Uf3SpgP+RO=dsSPXPO(Tn z%*8dP{h}D(vvu5r7%!&9!$q?g*4PYtB81l>sezWanQ^1R;%upXQDCXQPq92sFGI^0 z_mcrIFEjTNz*!HM#mH5=@w3-9UHLm9$3uNh!9>&K`*o%`h8JL5d(*!etx7I^6fky2j^Y~8#a9@KD`z9rBC^=}h#b+y1Bi&^h@ ze6b`AUv$3XArbXYEwG7#)+1&t_ai=OR{u!`)9@s&Q7jySiQS7!2&dbsLL0{>;D*ZZKwVsX3GU|F|y zq1B?=H5OA$u>{aV;fcwA-t9dL$n;D!Tr&qcyT4Z`6BM~ zT5EyjQF)!sDZ)wxXDI1UM<=*{wxH2EQ4&vcGWMN}&1w4#%el~;;#v|Z^);uU(TaFb zPWRvJhUS!{NW|6EoT7vE@yHZYpQ^$Av$oxdsDElsU8hLqKy&J)M?~^Br(H^Ik=aGZ zpewAXb;d-HU`277lfn(wM{L0Pp!v|8(#?pF1~#X)gMZ+Y`&%XwVch1lv}t=>n3b>k zZG`3&Ifn?SDYx4!dNc<>>Dgo?tm@ON^&mI9KRcA-t9mBhk{~rM?9Guv_4&YiDo&{~Hzd%m2Gg5mCgmw!o}c zTJ7QfS+90P7`FxLv|x2NxrOucU}0YEOa#=?0>9?7ZV@;380ZQuP^mkSByJ1j9H7OA zqUO*7{d9@g`di?aR^71$`fZfI4=vDP01+rw3;cNh2`<|AL34^=$5R?XWb*D3&a7Qx z!>%qDpar^(C1M)Z0(d-bjh1v}1_ZIs%p649e^ntbBMTZzq-#s~3 zf(f&FxWajf?MH<0o);r?+&vq~j>ao=^5?FP1(xWY6iXk=M`(Fi^xG9I1DmLc#*1n3 zFZgKIy7XL#OOLXc++ycrbs`mzq)Kg--n3m?A`t8B; z<^>VLYmpw0mhlx!@UNyJCIOQ@Yu8xK5sD>n2J6=$k=@P?2aCT2F`jPWG$(DzX3k*A znMQ>0T57D&qSgH2SGYh)IZQn*upADrvpI3TdAt&pAp!T#T6z#kJk9CltKK-zJ-_X1 z0nKUnDk7!6=G3BbC;SL}OuOI>&FP>Y5m#4px-hak9$Eg+N^gmIfwp(3mAbpF0|F)YlENkl*$%}H{5KV0IA4t4i{<`k1kB#GOcI0x8g;3|7) zPLU6Z*!r8(!go5@oKm}P$%W=LKc5H`t2uQyD!@hCD)P%L*d=nm5t+QZ#Ni2e3PaxZ z)_~^pK+;;YOEj!GRi1l&AFc#L&W$S;>^SAeD7$BPg))+wea%aR-LrihF`mAAa+*`e zOcf_MFA>Q^2=93*pNQwh@$pg{u$1RL8tqkkB`7>eu_#_+{eJVS%?f;$&b){iFQ!Gh z38=vZtI(Fy{&=I%ji+Wc)Er2dGS(O z`VB0)6NnIAOTrAayjQtk4VG`I_e%tpi&rU@2YdSBXWwY^suy61d_s&D(_+8VZXsA= zUl1X@mW#8{GTM1i6bL48?~?RYG&RBH3S*#M7KcJz+f= zb^GF?P-spi3PehM&8hfj2Rx{RVFk(1oW67=;_7Nn=ZaWI7W=&Y5ok_*^@*r|YED-U zJdlRwv}6bo$={ro>{3GJ%}?)-!HSwbjtCO0C{A;Va(;s$&(z*WL34UBod{`Qb1EMA zdK5ILDUL)Kw>jx`XMMf@r$w$WEX<*9L_i(Q$!j3%cK3?iIkwQ8B)y3wahnt80Q2L{ zl|pmU_90^HZ%+AoM%bL%1ym(MbF$k;1d7$1v@X8Gt>#>s^j_E{d_#y#-d$pc4ek;; zgS?JGb1FVT#5AlqEqalR-#jk+{C&D$$2sx;9f$Mgi?_k7drk%NTb{vMf6(}cXgq!Q zpbh!EcMa)NbUT!RKm!j)i}ic_}05}HV{Xl!DAi`t~XaW+^U_aMfL zX&Knm@-0{b^@tE&OQ&1zg~(_iYg1Iy>3#CW=e zbI(chLknfFI7}o$cr8J5(IRcqSrsh%wg&GISk|XeEGv{*Z&>)@J1z?>JNFUe=@w24 zoNiHk2rQSvh!9@O`gv%nP@d`wmR*zmss)znbc$s}57zT%hdYh#1eR%T#CW=e(*nQu zI<^KZVT;y_4uaRRY(82JNXb@#WyaIcQOj!ASZNl;vPNwvuCbYhiukXvdwn9t(=D9O ze)A?F`0qTvYIaa`5WE%_C$t#7?1k?>eAnw(n!r-?fp?`feUSkxVl=Pp*-s!u^BEuGzMDWcPAq1 zpIV@*LVyOez#MlXlD`GEQ)ZRjHS;&6!iw6vf(R0RWP-VPQg@1)xTOKKj)a;+=t%l;GDubRN;wv`_- zo^Ii^jp6#X_;!=SK|~0zWt1ygejd1r_cf(U4)$45J1^G@D3%plSdDgi(GqOuE31j| zbPH!*wm^zuI3bcK?#pcRok)=G^e9I ziKu^SPMvpL$9L{`=|e>FHz&V@j>x=_RM-|i9sP`nAi;{_G^bI`USY^#{j*ijoX(6S zLK@hd;s)zPKy$h~o(SVMr(1@sM@Xz&`Fw+gd3-t%P)Bpxx`y>N#5DWOe$brUZHXjt zn-k{%O^UsaLUTIjOvKjToQ|1~!R9pCI5rxZ)6OMCpjgdmcV~%V@WO7NPX=pYm+VSqwO2LYN{Yp+@*;lZ25sGr zzu~$yjTle2aN36EGJCvFGr*n*;k8JtKufCv>)$XhJ&qJ@6Id$0Q7ox%S@$*1HccG{ z^Kv+f7*DrwuF%h_CGkFu(nTVK*HXC>Ee7!y7l1{n+uSy*Yv-lpC&hBO(=VKtIqe?e zzxMrIn;1{GaOTC_Y0d=r>^(FjLU=7DtI;w$ef&VMv{}E&T42fhL$TZ$*cGo^=ho`& zhk3EsMvSLhIBg?+N#As^xbGuEcrAH8Xz^8?*#RsO2IffuOO~G`_ul!M$L46!yInaD zEKdd#mF-r*I3FXip9&}2CgwR%@F+6 zPSH|gJl(?iytsEB`T#6T*AgMTme*_1GUBpIFkEcE-HVzau$2GbBD~mApWH94V10`k z`{x=yxj!L~NaAUMI%=$cKOkB)MH5=!n>R#CeJ!w8L}xtJI?wYIparh_NW|6E0?mf8 zo2Ur~3J_B0dgP}xh{Vni`$|P)oZKZb(gcj&Kh6og^1@<{3 zi7jwYt~I{*J7Nlv$-7HbvvvvX%;EU^%{W^kreQ6x#qLA+hJ`5+QL%y@=gJn!?s-+A zHC_qU&Ym?I)_Rk3#CZDd$+@aYyeK^hyJwecLs@uh#U$Z|igh665I>PTT0bXZta*sG1QWyq2eaXo-mZ?G9J97b;Ku1(vW~ z6w6i<)-RjX3X&hfyr^msIg>Apk=uO(|eT4H@Qr-DV+(Y;Dw*}acq zQOWehHP-FkI}@<%^d`pBEmeR141vGErk{t(l91?wst)mP)8PKu6a^xQr#TI>=!R?dl~eae(3}!gh?M%8(~!NaZyvXv%-#jf zDXJ$CS66d#v}C>0w)m}AM`%v221L|9HK#!@l1D>xvK~rA@;9e*R;(W=90`y804wVE zC?ZI(qBzaz{XEuJIuBY^+=J#MKZyuwU~~F-@q#}zr3*g_xv#;?8(zP7aHSBypP)=KxQr>b8UC)N=(9TYqyhTsI4wQ;^AHA81Zx zeng;H&B^|N3^u2#G?O>5ODx__Wb*A2zHgtoIQy4Qoz@=jO%0?%6ua zZuHvPD?#~D%I=xzejE3hm7$?hu-22F6XWT-C+A8q;YsQkI4=_`h!DQ>B5{RvUTSP2 z@eK>hckS3GuoOm8EYD8Fq9sW`_#{|zj}qhQ7S7+$8`UpIfMr265yERZzZvKFpakm) zap|+Yo2;vym+UBtMe{Q2R|u(oGW)rzlhH` z7WySS2`uSnD3+KBtT)FNA25FjmK)oM@pKEP(LSl(H4^6K&pslA*Alq}EnPh}vA(TS zeHJUQB%G&M+&&J+E9%iP@~y#=tVN8cTR3a1#iYR!VDacngz#Fz0?}gSZi#PNp6cge zi({+2R}haE6vT^)iGOnCMO<@w7mz6|CVN^G(|nkR)iLKd>0Y*Pc6{uQkE>V!1-ZBB!3Gmab~r^0Y8i# z;nN`*MFa^}6sH9`C$U=Ky}09Dp#}bkB|;k50=KQyk%tx-oJ53iTcGMhM_iaq7iC+( z!n}Bo2&kh4N;!R`nr5crE@xXqnkp z3BN_{TAX%QU|D~gVk!S>hx=pUrT1;X^1GZEPq%QcYF!pI$F~yn`ay*7TGs7AOK{`> z{N}Nv{&%YlwQH<8jba()+!ieXiW@G1WuGxIo^Ii+vF@Gy&%tMKjb-|6#@`4VIT7RO7S6oPFY&^EdFkR}B81oCy$db7vZ@!u z=S4SsagD%I@PJ|oT@;NL#qY)VjAO=WV!W7^%G_O-VP1Y;CPH{E?z_=a?wITj7pRZv zlO}AeU1PqF>TFIC?yM*G^=G%th353@Ig!NEoD3hxx@h%RJg*b1MDf)EIUNuaQJ1?=F$R+9iCl-yenMw9=Vhhy4=b1#87pEu-NI=byQ6g*fo0wdB81m6Ef_7n z(@%DTHTI&jvWkE0yhOaESZ4fY-9HckZ{+BSLsB6GPA< zWAb=ESO$NdzeQjPen+v??7Dz!OwH=M8d%aK9Yy2m7S1=q3Jc%kuXYMr6Cu17i~VRh zGIs4kxIk5Xz4%jLSyxtPbJ{*v7gd9FoKm1U8MKZSCGj+;Gtq{)W={nrEr;gRO_xZi zuQ~Zvu%6lKakF_6G^ZV-h`73%Q*SHQ^A(MsWxs^x^k*g!^-s;I=gJ%2(41O$5t01O z>GMNYbMjX^)*V)qRR9qrSW%qjG|Y^3yWMBkyx-8A+Fv3<8rYmNT7KUR&B;HT2;(-V z(+gM+-^%*^dIk$q_a_lhM{^od$a?4@qklmuG$&ofOQMA-Zgb)sp!K(dZJ;@AF(6{= zZ%+HGmtk|-((lqVXinx+h(NKLlVfZLY)-mEW%0>QtHnen?=CUy0PYe$dVKVQ=9ImL zh-p}J8t8CO4tCEUpHJ5acAUWf?>L+{UvxH!!hL4p_^J5x?&Lqjc>3l` z6J5yERR3B@^{Xmr*E=A~%XxfFq=vXWvMcj6w-%h{~2yTB4OpBPWKa2oB0PBt^Z zqUA<}@LC2QL`%zbgEFwZGJT^HP&+Rr)fCIJjsYv*x@HC9j5JX?>9OdGS3lwI^7HloI2`v}{ZpKN>7$l|%@y zMdvVDVnaL6gn3!^IpnRtlI7c!``PdCo%OxX$aC&Vk+Y=sx$?f*t3IKV|nEckwhX({Gb^o`AJJ;|MXHzI$@6 z1Z&5s#K2YU(@7$P_q=pIio56GN6|gOQs8)hPGIeoAUcp@DV=u;ErlDBumxWCCdSh( zoGU>=nN|r{lKqGfUW;NlT6VTBIt&)w7eliImarg-rTt*mH^S{z&*Qt?E|w7E=@!nt zjYjUS*f!c%5Fxx4xd^npi?zhJcuYO`u;bR+dD*?2Vv#ms{VVMK(qDCeHKq|wjHg>T zYs~0If(FdXq02-FuSND4T2>bAz%^#ObaA-A;vY=0xW=>IC6(t98xIzT&BS=Rh0|!| z*006i)lS_{gz#DjC>3+Qw?^%a`am#IV&PjmV< zR~`FFAL*)7(44d#h?M%8Q?pg9C-+NtXpMp9uE${LLw@2dg=)md_puE6RE=5hPeqoaW@`$NDEa-X&~EfabI%f(U6~ zbNVo9@jGZvOQMM|ZgWygVEv>-L*WqK*gbTC2&khu-P*u<7)9H>iz+myJJ*OLahnt8 z0ApRNyFqg@PbFgOZ%%6_ZouYrb9?JxXilph5`khhrvyn2Y)&y>Lps9lz2Xg#$-7G& zKaM+IuP}FaXihF4h?s^orv&wRX0UtqcirJ6*m24aQ+ChAeyndFwkN%L3Txec6)~Q^ zdvdPO$K}#9;Ji%ULWJ<1mul8|(LVZp8dyRqpJodzh2a#-+r>w5&zyh3dpuZb8a)$@ zr&~Bz0!^io&tSRNiU{Ggd_IA5Jl1z0K8bv7oMNx-wO4}d;}lDjGVA%X_7aBIz@jvV z7%!$JB2=LTSc=_<5ME2^NwmC+vrLA0xpO4LU=6&r_dr-rI8O7vj@{=2i2~zix()C_+uG3FZaTaS}@WRS%%OpRgVhIv%=P4q9OMVj@Z07RWik zm$x7JK?}_OLd4eJ0u^s=#ug|uFvt*EpxrMbP^=bsE?gU1V6pwHv9L>IHP;b+ym@zt zU#wk1F4FlTv_NMCBBo(2u*Zugr(yT>j`%ZfN9~m$G>)=+DlI;S`^=q|%kfQ^-D8OH zV(y*`PL2wM^YY^=5yE?3GSA@d*?sK!!C+}Od0mdc5_FSd(JWxKjk8xz9)K&s_j$y4 zx`opM^Sk#+21~dX5yERpJ&P8<(8M!fiEAFCwzGC#)+bXe@;;F`FRjgwVmtR%B*xP% zoU2+^;&Oc3>c_4`2(Kk61}(jJJ)s`?dDkr;|Tmy&n6y*V^y(JZrDL&YvaFFVQndh$KsFB(ki| zn2IOIMt5rRRbVN*%V9a1uo7p~{14ye0ZTJo(w@&EIxQlqU6~QE*bE>ck}NS%$Z|Su zhAs3<<46ZHANd$7Nae8ToH&jB^81k)c=lmHJZaBo5%tUD%4^BM@+6IfNU}s+M3#2D z^CMt_s<2xhB(TJ%DKw`4o<<4ME zI6K`<6%4}WgM^!^Ca#0EFyCn_QRsPz%o35gh;Z4T*5xiEz85bL0cyH z`U@SCU2j=lrI?68@NkNV0feMwY4%Y0QZv<-_g^ECEFv zmVU#Sx7+cCD?S5DXccMCXAwOwEj4qN0n6z|CbbtK$>M$mS-Lm-^)~d2YHl0NgYtfH zFXgaYZpZuroYh&aH?UN{C++zxqIFFdA4lBdc)EdR?L|nk?7oUDHooIDVY1zQccq2E zQvN>{1?y*texvZoiA&h}4J^>>2}zQ$zyo_4;+Spo^_e|b;FD64QkMl*CFtNqb${1q zB3NM5ClaT~0zIZPZ&44|G&BVZJf><}d#Aq30$)#eSO^yA*PKL3Ti~v(45m)zV|*{4 z+l~YYBT8g}o=F*auO7A}W@>i&y#wSR7e)SZs2fW*KH3#m?jz|(BbzfARu3`1ejk@#!Sj<=7 zY2`1U2|j(`u$(#0>=54`YuXW5Ht!|v`7EMY&1hi}o=Wf_goH@4Y>q{iYc4Ox0m}rR zVb%gm@n;Upa7$+W?BTuMGlAt$AZgEM5t);I&Cp+&Ozl*xV0TJUgy@ zDzN0qI4u8`+u|&AJkbQtlbq6nwCA&k#@O_xT3NvIj~)q;WLXu5Ec!PFECrVI?9IIb zpHNRqPVB)Y)W>T3F6PRkCG*t*T>{fYoIr`BC| zyasbx7eoS;GN-jZ!_k~7?LF`$&hjvlDOn|+FjXQ#EeB7ImBo>m`k9k^lvgQK&*tYY z>Ku~K1ebSls^=tkOPmQ7+k|;T^}H5C+ViWYXeO|U{J|2cXXXtOB6(k|6R>(tIAb3S zEGAzJcMB}%+&C;LivsYu-5XTV4_GcYFsg0OXA#-P;7OY`fMsb*5+cbm<0i6H&uv!) zEIU%en*Js4m!SO|mho+GB8!FZt}ei`C5N=w+dWz|!LoX|E(ptahsd&@bE1k`PIj@wbr0c?U6R%7-B%Kq7YA6Ps}NP9kuXp9xu`8os3n2#hxlEv&cvaFrbC=yt-C&>C7 zmXEQ@01nGUOXioxq3}ldOC#Qzv{#ZvL#xUTSYEFoA(AX3l91)ptOcWh#jEKvJ1v#_MEmOK(H zaJ>$RQ)GeJJ((-p>Xa}LEYP(tiTW-JTr%&xEm&aYKoTi!f&FS23p6ow)qoM@Z$g5E z5hb!fl^SOK>{H*IcVL0lV@XIoE$~9ees916GpCX;u>~&3U{<|yvbq+)z&vY30u)%F zrXI8P)%C>2Vz9uC3rUi)7AU&FxcF($zycjslGwT}u(DMkTA=C9m>yt(-!_myr7TeO z+Gw;uFYRtPs}?wuOsOj2WX4p926;~1;VtSbcM?-S3v?d41$XZjnHFgrkhjMcN@a-Wef6ZDU5I$vPP z3gNKa?BIqhPfztb3M|L&lJ!;2 z*=f)(>7Pk^K8wf#Z&x{Z0!vUe36W&!n1U>?4(`FT+}aF$S1quFMsrv~r!p+7Z_M2U zEJIC5dp?Wkmqt_d6#S-r#uySJ$@1fUWbvA0wGAeyV?$1lJ1QSz{xJ&8$-rPN_K>l| zjX_{e^UO$+ggKQpWZpbl*8F4x=Jd-Hl2VsB1y$+dMd?~)=z%%q%ph@!%*o~%bIPOY z@kgt`oSYVtsP8hT1+x}+0dtzPoJ2~SQ>+@Z%`T~b;$9e0g?1!J7*Qf~vQy8*d)0P* zn>k=kx?4#|Jd*BtU^Vb=}Hr`PtLNLE1as02B(Zgylb3HWn$wi8XJuecN&owd^8X)2rOc`2RdY0_H~xQ& zhKi>iK{BPPgwsBzN)%6dTMg!3ruB zH5lvH_mlSg>M62~h^})xz?FwFYhc#dp?V3R%<=;n;)>aFDD_AEDh3-rTeP3S}?{s?#m4qSSk`ZEM`YcaE&&* zU{fDp8Sh8h^I1gwVjH_K6JEFyC{x?$ubV412#LL^zfJj8w(Aq!gs{qi9*aht$WaF4@sGm80N)GqE9 z7el{{dr8{!Sw#J!m;UxCu!MgkA(AZbA0f;As5UZ~pk&S`-w7=Lr7AS1FQLrI7oofI zhJ!ho?k7nS=A>RR8pmwvqZ`Y>oU{*Y^U`}H$lSpZEvi4-mDfQ5J{JnSNCJ7Qol*pWZpTn#UC$$b^I)FK;H#}AQC@N)6!IvkYIR%Xl#~tck ztx2X-m2kS2j#a{KkcB>&Q?H*$O#RI1khiKgRL@&}SHB1qC*uL9dX9{nh|i(L!yhj| z^}KhKwC7h((M+&mi9u_qo~oxwh}3;iDP`_U#n`>Ez|!OG;FZDhSuHu8!xD3k`I3Lz zhP!xrq(L=luOy2_pPpBNdzYjOKc{G<<3avOMc@* zoowiriWQ{2k}MkEp4kA)%Pk~Ck|qB!vWy!L-W>YHr@@^O$L0MJk;`Fe9<7FBY^-(h z0ASH-LE7_KM0P%AofW=$9NdnCNU~)8i!9|2FUWx9+0OyH1eTDe9G2u3dvOY!@^s!Y zV0kx~wCA&ko)QMku(_81Z($?_-@SvDoBXaLLOk(H$aOF$uqWze!9WEpO@&IMTZ zOd{?1ETVpSDVvG=0{hJ(A(AX9S;#W+qG|-NwEn}}@`QYhxxeDDXr+w9G4}JUQFx+n z^?cHv&m!uVkM0}I;d!~fj)X|EBxNIu;f;Y`VX}SqYNWrw@~T*|1-hDI58clnoBxg7c) zEbv+r67^jcSXefF30R;>YZ57Kfvt`(m_8%UIl|MiygdmLMwG|`RZ18OoUmX23|OF@ z4hgBJ1wQHF)Cnx`T5l32w!o@<=0p6pM|J1Hzzpt30u)%_cx`5T^{?h_y}<(a7?C7p zEl_lUU4OWp0v34Kgv8cufqk9B&;mWOZ`y+e`k9eHr7SSaWg1%GWc7IbR>^%L$&{)R zP6jzxC6cTfP6G?P{RfGup9TJDU#QzF1`~8EH{54A(AXtbFq)#-ts5|mQau3Ap%SBM-Iz%jeKM|sihGPEPh8wdp?V3 zR@2>5g=C;Z; z%rLda2UmkRwOK||>N2M?LuO&=O={=GU`{jFk~l@?l+%(ir{AhR90qgJbR<#VWlmZl zeRIK_3f)Mgv^f=-_P~3;W^kJf7*QU6BuE%hB6B+6{0KuFjGKQ0b214cA@wvTd%ZDB z!JMX@BVl55I$p>eD$t>G-ZL1ORxu<%fjMkmJ`)7(<*=kW_+j~QyRaU=t@Di|?UiI%{Ndv# zU|Et(LL^zdo+HckOS-RsCG1nH{Q`@x2ZzN!otdFSvO4)dzwBd@?fERCF}6{=0q%C! z(<330EG{pQWx8V~zNxp5(U}!0A7hn%9G3Yu%)X>}nFp>AoJc0^`7EM-Nv`OE|H5AV zn1o2OY=4O?r>#q0!x;M(;SwUSyg$TYS?}V5&r4xTGgTO4<~K-tK8wg`%Xhz63u8S;M%v;pdnT_!+Y6l$>r^o{L)pW(D>CSqOO0d8oeM!`JS)f^Di4$1h zP9qX2ZGk@R84I*~q@M^Qs`&^KB#bDL1-jZZC&z~8#5;inR*fei^|Zjru1DR$0=xY| z!o(Ihq?9>cX6%@~{xC2#=8ymd7Fd(Sd^FS~bjc}LKWnp;Bq?iwq6>_l?VS%6*kl!n zt=j@y$}XY>+9eEe0}JfwKmwJrK#k|~&;mDF9x8w;k+qFvN>+&?rb?`vZ)*=0D03k( z^|L^8{l%A{dfu%H);%Mi30?$ps^_tB%zA+PIh4C1f>e&u02qCHg#vC8!&-+uc;-)&yY5SVY?MSwzoE)1hzhMBnxHBt(*B_G@G@ z{2b#0{nF=!YumH(ehH1_u>8G$AofdsTpoVuva^D;=d*~u-L~yA^f^5HZq40lFG7-K zS{bqwss8N(Gu*NV&T9k~m*~cQ#53IDcg&F{6ZgsRzca@bkpRi-Xz&KFqvXf~N2tw? PUr0>-YV%(fVKM&)-|3S# literal 0 HcmV?d00001 diff --git a/logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/trial_1/flags_0.json b/logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/trial_1/flags_0.json new file mode 100644 index 00000000..f4a145be --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/trial_1/flags_0.json @@ -0,0 +1,63 @@ +{ + "logtostderr": false, + "alsologtostderr": false, + "log_dir": "", + "v": 0, + "verbosity": 0, + "logger_levels": {}, + "stderrthreshold": "fatal", + "showprefixforinfo": true, + "run_with_pdb": false, + "pdb_post_mortem": false, + "pdb": false, + "run_with_profiling": false, + "profile_file": null, + "use_cprofile_for_profiling": true, + "only_check_args": false, + "op_conversion_fallback_to_while_loop": true, + "delta_threshold": 0.5, + "tt_check_filter": false, + "tt_single_core_summaries": false, + "runtime_oom_exit": true, + "hbm_oom_exit": true, + "test_srcdir": "", + "test_tmpdir": "/tmp/absl_testing", + "test_random_seed": 301, + "test_randomize_ordering_seed": "", + "xml_output_file": "", + "experimental_orbax_use_distributed_process_id": false, + "experimental_orbax_use_distributed_barrier": false, + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "workload": "imagenet_vit", + "tuning_ruleset": "self", + "tuning_search_space": null, + "num_tuning_trials": 1, + "data_dir": "/data/imagenet/pytorch", + "imagenet_v2_data_dir": "/data/imagenet/pytorch", + "librispeech_tokenizer_vocab_path": "", + "framework": "pytorch", + "torch_compile": true, + "experiment_dir": "/experiment_runs", + "experiment_name": "submissions_a100/ademamix_golden/study_2", + "save_checkpoints": false, + "save_intermediate_checkpoints": true, + "resume_last_run": null, + "append_timestamp": false, + "use_wandb": false, + "profile": false, + "max_global_steps": null, + "overwrite": true, + "hparam_start_index": null, + "hparam_end_index": null, + "rng_seed": -83011591, + "set_pytorch_max_split_size": false, + "pytorch_eval_num_workers": 0, + "capture_jax_trace": false, + "skip_evals": false, + "?": false, + "help": false, + "helpshort": false, + "helpfull": false, + "helpxml": false, + "tfds_debug_list_dir": false +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/trial_1/measurements.csv b/logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/trial_1/measurements.csv new file mode 100644 index 00000000..640e3269 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/trial_1/measurements.csv @@ -0,0 +1,383 @@ +global_step,grad_norm,loss,train/accuracy,train/loss,validation/accuracy,validation/loss,validation/num_examples,test/accuracy,test/loss,test/num_examples,score,total_duration,accumulated_submission_time,accumulated_eval_time,accumulated_logging_time,preemption_count +0,0.3557709157466888,6.907755374908447,,,,,,,,,,,,,, +1,,,0.00228515625,6.90775634765625,0.00176,6.90775625,50000.0,0.0013,6.90775546875,10000.0,121.32478308677672,388.9585015773773,121.32478308677672,266.90091943740845,0.0,0.0 +1,0.3725830614566803,6.907755374908447,,,,,,,,,,,,,, +2,0.3841941058635711,6.907754898071289,,,,,,,,,,,,,, +3,0.3709340691566467,6.907753467559815,,,,,,,,,,,,,, +4,0.3667190372943878,6.907754898071289,,,,,,,,,,,,,, +5,0.372001051902771,6.9077558517456055,,,,,,,,,,,,,, +6,0.3810482323169708,6.9077467918396,,,,,,,,,,,,,, +7,0.3730814158916473,6.907751560211182,,,,,,,,,,,,,, +8,0.3669123947620392,6.907747268676758,,,,,,,,,,,,,, +9,0.3698151707649231,6.907750606536865,,,,,,,,,,,,,, +10,0.3828930258750915,6.907731056213379,,,,,,,,,,,,,, +11,0.3619360029697418,6.907731056213379,,,,,,,,,,,,,, +12,0.3728687465190887,6.907744407653809,,,,,,,,,,,,,, +13,0.3725390136241913,6.9077253341674805,,,,,,,,,,,,,, +14,0.3689760565757751,6.907753467559815,,,,,,,,,,,,,, +15,0.3744933009147644,6.907721519470215,,,,,,,,,,,,,, +16,0.3718143999576568,6.907721042633057,,,,,,,,,,,,,, +17,0.3628654479980469,6.907713890075684,,,,,,,,,,,,,, +18,0.3788138926029205,6.9077253341674805,,,,,,,,,,,,,, +19,0.3581025600433349,6.907733917236328,,,,,,,,,,,,,, +20,0.3654921054840088,6.907702445983887,,,,,,,,,,,,,, +21,0.3659215569496155,6.907679557800293,,,,,,,,,,,,,, +22,0.3783256709575653,6.907694339752197,,,,,,,,,,,,,, +23,0.3669207394123077,6.907649040222168,,,,,,,,,,,,,, +24,0.3771553039550781,6.907607078552246,,,,,,,,,,,,,, +25,0.3648486733436584,6.907609939575195,,,,,,,,,,,,,, +26,0.3815815150737762,6.907580375671387,,,,,,,,,,,,,, +27,0.3803050518035888,6.907617568969727,,,,,,,,,,,,,, +28,0.3782521784305572,6.90761661529541,,,,,,,,,,,,,, +29,0.362415999174118,6.907675743103027,,,,,,,,,,,,,, +30,0.3864795863628387,6.9075026512146,,,,,,,,,,,,,, +31,0.3541291952133178,6.907658576965332,,,,,,,,,,,,,, +32,0.3734919726848602,6.907576560974121,,,,,,,,,,,,,, +33,0.3676976859569549,6.907505989074707,,,,,,,,,,,,,, +34,0.3714061975479126,6.9074387550354,,,,,,,,,,,,,, +35,0.3625860214233398,6.907491683959961,,,,,,,,,,,,,, +36,0.3885780870914459,6.907403945922852,,,,,,,,,,,,,, +37,0.3733708560466766,6.907426834106445,,,,,,,,,,,,,, +38,0.365086555480957,6.907397270202637,,,,,,,,,,,,,, +39,0.3534780740737915,6.90748929977417,,,,,,,,,,,,,, +40,0.3971766233444214,6.907308578491211,,,,,,,,,,,,,, +41,0.3674996495246887,6.907305717468262,,,,,,,,,,,,,, +42,0.3867277204990387,6.907255172729492,,,,,,,,,,,,,, +43,0.3939203023910522,6.907198429107666,,,,,,,,,,,,,, +44,0.3887163400650024,6.907016754150391,,,,,,,,,,,,,, +45,0.3427172005176544,6.90733814239502,,,,,,,,,,,,,, +46,0.3969490528106689,6.906972408294678,,,,,,,,,,,,,, +47,0.405188113451004,6.906807899475098,,,,,,,,,,,,,, +48,0.3676764369010925,6.907041549682617,,,,,,,,,,,,,, +49,0.3655986189842224,6.9070539474487305,,,,,,,,,,,,,, +50,0.4025768041610718,6.9066925048828125,,,,,,,,,,,,,, +51,0.4006567597389221,6.906704902648926,,,,,,,,,,,,,, +52,0.3488681316375732,6.907064437866211,,,,,,,,,,,,,, +53,0.4127519130706787,6.906449794769287,,,,,,,,,,,,,, +54,0.4008476734161377,6.906489372253418,,,,,,,,,,,,,, +55,0.3553452789783478,6.9069108963012695,,,,,,,,,,,,,, +56,0.4082881212234497,6.906373977661133,,,,,,,,,,,,,, +57,0.4221036732196808,6.906233787536621,,,,,,,,,,,,,, +58,0.4186397492885589,6.906149387359619,,,,,,,,,,,,,, +59,0.4256295263767242,6.905941009521484,,,,,,,,,,,,,, +60,0.411630243062973,6.905988693237305,,,,,,,,,,,,,, +61,0.4293310046195984,6.905457496643066,,,,,,,,,,,,,, +62,0.390190064907074,6.90602970123291,,,,,,,,,,,,,, +63,0.3858069777488708,6.905623435974121,,,,,,,,,,,,,, +64,0.4157457053661346,6.905159950256348,,,,,,,,,,,,,, +65,0.396843671798706,6.905806064605713,,,,,,,,,,,,,, +66,0.4364452958106994,6.904608249664307,,,,,,,,,,,,,, +67,0.410207986831665,6.9047441482543945,,,,,,,,,,,,,, +68,0.4438543021678924,6.904394149780273,,,,,,,,,,,,,, +69,0.4466149508953094,6.904644966125488,,,,,,,,,,,,,, +70,0.387826532125473,6.904870986938477,,,,,,,,,,,,,, +71,0.4621224105358124,6.904311656951904,,,,,,,,,,,,,, +72,0.4667878150939941,6.903342247009277,,,,,,,,,,,,,, +73,0.4648320972919464,6.90384578704834,,,,,,,,,,,,,, +74,0.4626313149929046,6.903715133666992,,,,,,,,,,,,,, +75,0.3929626047611236,6.904351711273193,,,,,,,,,,,,,, +76,0.3972343802452087,6.904220581054688,,,,,,,,,,,,,, +77,0.408307284116745,6.903604507446289,,,,,,,,,,,,,, +78,0.4885356426239013,6.902553558349609,,,,,,,,,,,,,, +79,0.4817683696746826,6.904542922973633,,,,,,,,,,,,,, +80,0.4702984094619751,6.903170585632324,,,,,,,,,,,,,, +81,0.3955028057098388,6.904805660247803,,,,,,,,,,,,,, +82,0.4635762870311737,6.902248382568359,,,,,,,,,,,,,, +83,0.4479374587535858,6.902943134307861,,,,,,,,,,,,,, +84,0.4061595797538757,6.90326738357544,,,,,,,,,,,,,, +85,0.4054405987262726,6.903816223144531,,,,,,,,,,,,,, +86,0.4999989569187164,6.901287078857422,,,,,,,,,,,,,, +87,0.4776759147644043,6.901191711425781,,,,,,,,,,,,,, +88,0.4628950953483581,6.901599407196045,,,,,,,,,,,,,, +89,0.4103515446186065,6.902168273925781,,,,,,,,,,,,,, +90,0.4902635812759399,6.900558948516846,,,,,,,,,,,,,, +91,0.4514320194721222,6.900492668151856,,,,,,,,,,,,,, +92,0.4936517477035522,6.899875164031982,,,,,,,,,,,,,, +93,0.4803199768066406,6.900359630584717,,,,,,,,,,,,,, +94,0.4977109432220459,6.900460720062256,,,,,,,,,,,,,, +95,0.4999990463256836,6.899870872497559,,,,,,,,,,,,,, +96,0.4999989867210388,6.897614479064941,,,,,,,,,,,,,, +97,0.447003573179245,6.901553153991699,,,,,,,,,,,,,, +98,0.4838730096817016,6.898449897766113,,,,,,,,,,,,,, +99,0.475560188293457,6.898356914520264,,,,,,,,,,,,,, +100,0.4911104142665863,6.898347854614258,,,,,,,,,,,,,, +500,0.4999994337558746,6.733353614807129,,,,,,,,,,,,,, +1000,0.4999996721744537,6.415021896362305,,,,,,,,,,,,,, +1500,0.4999996721744537,6.233492374420166,,,,,,,,,,,,,, +2000,0.4999996125698089,5.983521938323975,,,,,,,,,,,,,, +2371,,,0.055859375,5.700372314453125,0.0493,5.87810625,50000.0,0.0486,5.6401859375,10000.0,2692.0896463394165,3129.844084262848,2692.0896463394165,433.4192156791687,0.2330889701843261,0.0 +2500,0.4999995231628418,6.518265724182129,,,,,,,,,,,,,, +3000,0.4999997913837433,5.5807929039001465,,,,,,,,,,,,,, +3500,0.499999463558197,6.144539833068848,,,,,,,,,,,,,, +4000,0.4999995529651642,5.228029251098633,,,,,,,,,,,,,, +4500,0.4999996125698089,4.978370666503906,,,,,,,,,,,,,, +5000,0.4999995529651642,4.987598896026611,,,,,,,,,,,,,, +5500,0.4999995529651642,4.651830196380615,,,,,,,,,,,,,, +5610,,,0.189765625,4.9383306884765625,0.17356,4.7599465625,50000.0,0.1741,4.29425546875,10000.0,5262.350856781006,5854.4937126636505,5262.350856781006,583.4502947330475,0.3073396682739258,0.0 +6000,0.4999996125698089,4.456670761108398,,,,,,,,,,,,,, +6500,0.4999996721744537,4.458788871765137,,,,,,,,,,,,,, +7000,0.4999995827674866,4.190932273864746,,,,,,,,,,,,,, +7500,0.4999995827674866,4.195642948150635,,,,,,,,,,,,,, +8000,0.4999995529651642,4.29966926574707,,,,,,,,,,,,,, +8500,0.4999995529651642,5.951800346374512,,,,,,,,,,,,,, +8777,,,0.324140625,4.036053466796875,0.287,4.09793375,50000.0,0.2762,3.52323125,10000.0,7834.110359668732,8572.682470083237,7834.110359668732,725.8568375110626,0.4261224269866943,0.0 +9000,0.4999996125698089,3.829831838607788,,,,,,,,,,,,,, +9500,0.4999995529651642,4.934957027435303,,,,,,,,,,,,,, +10000,0.4999997019767761,3.537519931793213,,,,,,,,,,,,,, +10500,0.4999995827674866,3.91812515258789,,,,,,,,,,,,,, +11000,0.4999995529651642,5.317974090576172,,,,,,,,,,,,,, +11500,0.4999995529651642,3.142411708831787,,,,,,,,,,,,,, +12000,0.4999996423721313,3.1661536693573,,,,,,,,,,,,,, +12004,,,0.40556640625,3.770957641601562,0.36128,3.826253125,50000.0,0.3526,3.036040234375,10000.0,10411.95853638649,11298.288755893707,10411.95853638649,865.246161699295,0.5769710540771484,0.0 +12500,0.4999996423721313,3.825783491134644,,,,,,,,,,,,,, +13000,0.4999996125698089,3.133309125900269,,,,,,,,,,,,,, +13500,0.4999996125698089,3.378946542739868,,,,,,,,,,,,,, +14000,0.4999995827674866,2.8920392990112305,,,,,,,,,,,,,, +14500,0.4999997019767761,2.9115700721740723,,,,,,,,,,,,,, +15000,0.4999995231628418,3.8914544582366943,,,,,,,,,,,,,, +15268,,,0.479921875,3.2422940063476564,0.41712,3.508525,50000.0,0.4116,2.7012171875,10000.0,12979.315444469452,14017.990199327469,12979.315444469452,1008.971207857132,0.8551476001739502,0.0 +15500,0.4999996125698089,2.651757001876831,,,,,,,,,,,,,, +16000,0.4999996423721313,2.6883697509765625,,,,,,,,,,,,,, +16500,0.4999996125698089,3.2438552379608154,,,,,,,,,,,,,, +17000,0.4999995827674866,3.995916366577149,,,,,,,,,,,,,, +17500,0.4999995827674866,2.80048942565918,,,,,,,,,,,,,, +18000,0.4999995827674866,4.223663806915283,,,,,,,,,,,,,, +18417,,,0.54462890625,2.800125732421875,0.46806,3.2255234375,50000.0,0.4518,2.47650390625,10000.0,15546.428710222244,16751.14104294777,15546.428710222244,1166.0138342380524,1.1592702865600586,0.0 +18500,0.4999996125698089,2.868805170059204,,,,,,,,,,,,,, +19000,0.4999995827674866,5.031514644622803,,,,,,,,,,,,,, +19500,0.499999463558197,4.902590751647949,,,,,,,,,,,,,, +20000,0.4999995827674866,2.429982662200928,,,,,,,,,,,,,, +20500,0.4999996423721313,2.529710292816162,,,,,,,,,,,,,, +21000,0.4999996721744537,2.420827388763428,,,,,,,,,,,,,, +21500,0.499999463558197,4.7179951667785645,,,,,,,,,,,,,, +21768,,,0.54513671875,3.107976379394531,0.49176,3.171916875,50000.0,0.4811,2.323434375,10000.0,18114.06486916542,19475.07040452957,18114.06486916542,1312.8532564640043,2.128077268600464,0.0 +22000,0.4999994933605194,5.043087959289551,,,,,,,,,,,,,, +22500,0.4999996125698089,2.342177391052246,,,,,,,,,,,,,, +23000,0.4999995529651642,3.81080174446106,,,,,,,,,,,,,, +23500,0.4999996125698089,2.2375712394714355,,,,,,,,,,,,,, +24000,0.4999995231628418,4.445783615112305,,,,,,,,,,,,,, +24500,0.4999996125698089,2.239621162414551,,,,,,,,,,,,,, +24895,,,0.56482421875,3.170403137207032,0.5066,3.2526559375,50000.0,0.502,2.215092578125,10000.0,20681.5164604187,22209.77625346184,20681.5164604187,1471.6794302463531,2.4110121726989746,0.0 +25000,0.4999995827674866,3.16884994506836,,,,,,,,,,,,,, +25500,0.4999994933605194,3.692971706390381,,,,,,,,,,,,,, +26000,0.4999995827674866,2.147036552429199,,,,,,,,,,,,,, +26500,0.4999996423721313,2.228750228881836,,,,,,,,,,,,,, +27000,0.4999995827674866,2.0943028926849365,,,,,,,,,,,,,, +27500,0.4999995529651642,2.367526054382324,,,,,,,,,,,,,, +28000,0.4999995529651642,2.396492481231689,,,,,,,,,,,,,, +28226,,,0.5911328125,2.883991394042969,0.5363,2.76262,50000.0,0.5237,2.1259302734375,10000.0,23248.10187292099,24931.695034742355,23248.10187292099,1617.216381072998,3.4876534938812256,0.0 +28500,0.4999995231628418,3.467546939849853,,,,,,,,,,,,,, +29000,0.4999995529651642,3.9469010829925537,,,,,,,,,,,,,, +29500,0.4999996125698089,2.2569773197174072,,,,,,,,,,,,,, +30000,0.4999995231628418,3.4546420574188232,,,,,,,,,,,,,, +30500,0.4999994933605194,4.515624523162842,,,,,,,,,,,,,, +31000,0.4999995529651642,2.3718338012695312,,,,,,,,,,,,,, +31455,,,0.663359375,2.111064910888672,0.5554,2.83841125,50000.0,0.5334,2.0450654296875,10000.0,25816.279580831528,27648.096289157867,25816.279580831528,1756.5567178726196,4.614113569259644,0.0 +31500,0.4999995231628418,4.479097366333008,,,,,,,,,,,,,, +32000,0.4999996125698089,4.621783256530762,,,,,,,,,,,,,, +32500,0.4999995231628418,2.121426105499268,,,,,,,,,,,,,, +33000,0.4999995827674866,4.300373077392578,,,,,,,,,,,,,, +33500,0.4999996423721313,2.13372802734375,,,,,,,,,,,,,, +34000,0.4999995827674866,1.7929970026016235,,,,,,,,,,,,,, +34457,,,0.625390625,2.8165191650390624,0.5694,2.614239375,50000.0,0.5457,2.0043236328125,10000.0,28384.168689012527,30376.761737585068,28384.168689012527,1908.323546409607,4.924053430557251,0.0 +34500,0.4999995827674866,2.012927293777466,,,,,,,,,,,,,, +35000,0.4999995827674866,2.045947313308716,,,,,,,,,,,,,, +35500,0.4999994933605194,2.01741886138916,,,,,,,,,,,,,, +36000,0.4999995529651642,3.2139453887939453,,,,,,,,,,,,,, +36500,0.4999994933605194,2.354304552078247,,,,,,,,,,,,,, +37000,0.4999995231628418,2.360288619995117,,,,,,,,,,,,,, +37500,0.4999995529651642,1.917846918106079,,,,,,,,,,,,,, +37631,,,0.65158203125,2.6507022094726564,0.57466,2.8467275,50000.0,0.5491,1.96443515625,10000.0,30951.68151497841,33095.715420007706,30951.68151497841,2051.183312892914,5.2484130859375,0.0 +38000,0.4999995529651642,1.8128604888916016,,,,,,,,,,,,,, +38500,0.4999995529651642,1.870051980018616,,,,,,,,,,,,,, +39000,0.4999995529651642,1.8200056552886963,,,,,,,,,,,,,, +39500,0.4999996125698089,1.9653550386428835,,,,,,,,,,,,,, +40000,0.4999995231628418,3.1804728507995605,,,,,,,,,,,,,, +40500,0.4999995529651642,4.405281066894531,,,,,,,,,,,,,, +40793,,,0.66376953125,2.469996032714844,0.56806,2.9299578125,50000.0,0.5644,1.912332421875,10000.0,33523.692170619965,35824.94554591179,33523.692170619965,2199.3028275966644,5.618121862411499,0.0 +41000,0.4999995529651642,1.9325108528137207,,,,,,,,,,,,,, +41500,0.4999995827674866,3.187605142593384,,,,,,,,,,,,,, +42000,0.4999995529651642,3.6043038368225098,,,,,,,,,,,,,, +42500,0.4999995529651642,1.7173466682434082,,,,,,,,,,,,,, +43000,0.4999995827674866,1.8578133583068848,,,,,,,,,,,,,, +43500,0.499999463558197,3.0470032691955566,,,,,,,,,,,,,, +44000,0.4999995231628418,1.8863446712493896,,,,,,,,,,,,,, +44048,,,0.66783203125,2.445870971679688,0.5822,2.6944565625,50000.0,0.5681,1.8966951171875,10000.0,36090.76889109612,38540.7880191803,36090.76889109612,2339.386500597,5.942543029785156,0.0 +44500,0.4999995231628418,1.7825320959091189,,,,,,,,,,,,,, +45000,0.4999995827674866,1.730241060256958,,,,,,,,,,,,,, +45500,0.4999995529651642,1.826855421066284,,,,,,,,,,,,,, +46000,0.4999995231628418,1.740220308303833,,,,,,,,,,,,,, +46500,0.4999996125698089,1.8603265285491943,,,,,,,,,,,,,, +47000,0.4999995827674866,2.0065507888793945,,,,,,,,,,,,,, +47187,,,0.61841796875,3.3203265380859377,0.58706,2.7905684375,50000.0,0.5785,1.8651451171875,10000.0,38661.46665644646,41271.61074280739,38661.46665644646,2490.3515722751617,6.298062562942505,0.0 +47500,0.4999995827674866,3.1819450855255127,,,,,,,,,,,,,, +48000,0.4999994933605194,2.264774799346924,,,,,,,,,,,,,, +48500,0.4999996125698089,1.6972126960754397,,,,,,,,,,,,,, +49000,0.4999995827674866,1.6126898527145386,,,,,,,,,,,,,, +49500,0.4999995827674866,3.7704343795776367,,,,,,,,,,,,,, +50000,0.4999995529651642,1.57802152633667,,,,,,,,,,,,,, +50500,0.4999996125698089,1.6794754266738892,,,,,,,,,,,,,, +50538,,,0.663828125,2.6336328125,0.59092,2.7246275,50000.0,0.5798,1.82652265625,10000.0,41228.18127465248,43997.95974302292,41228.18127465248,2640.9342324733734,6.681631803512573,0.0 +51000,0.4999995529651642,1.6456929445266724,,,,,,,,,,,,,, +51500,0.4999995529651642,1.650823712348938,,,,,,,,,,,,,, +52000,0.4999996423721313,1.753364086151123,,,,,,,,,,,,,, +52500,0.4999996423721313,1.6713805198669434,,,,,,,,,,,,,, +53000,0.4999996423721313,4.211996078491211,,,,,,,,,,,,,, +53500,0.4999996125698089,3.208651542663574,,,,,,,,,,,,,, +53663,,,0.66783203125,2.716911010742188,0.6055,2.58237921875,50000.0,0.587,1.7989046875,10000.0,43798.88992142677,46735.01598310471,43798.88992142677,2798.588620901108,7.0068535804748535,0.0 +54000,0.4999995529651642,2.467873334884644,,,,,,,,,,,,,, +54500,0.4999995529651642,2.04260778427124,,,,,,,,,,,,,, +55000,0.4999995231628418,2.784074544906616,,,,,,,,,,,,,, +55500,0.4999995827674866,3.385847806930542,,,,,,,,,,,,,, +56000,0.4999995529651642,1.7717959880828855,,,,,,,,,,,,,, +56500,0.4999995529651642,2.8060190677642822,,,,,,,,,,,,,, +56977,,,0.67353515625,2.700517578125,0.6015,2.7015803125,50000.0,0.596,1.776692578125,10000.0,46373.18920493126,49466.45277762413,46373.18920493126,2947.053974866867,7.303242683410644,0.0 +57000,0.4999996125698089,1.6452909708023071,,,,,,,,,,,,,, +57500,0.4999996125698089,3.180123805999756,,,,,,,,,,,,,, +58000,0.4999996423721313,3.958894014358521,,,,,,,,,,,,,, +58500,0.4999995231628418,2.3611741065979004,,,,,,,,,,,,,, +59000,0.4999995827674866,1.7355732917785645,,,,,,,,,,,,,, +59500,0.4999996125698089,2.986480236053467,,,,,,,,,,,,,, +60000,0.4999996125698089,3.907115936279297,,,,,,,,,,,,,, +60128,,,0.71884765625,2.027147674560547,0.6182,2.40730015625,50000.0,0.5981,1.76907109375,10000.0,48942.8839867115,52183.45779824257,48942.8839867115,3085.3943016529083,7.672607660293579,0.0 +60500,0.4999995827674866,1.7274320125579834,,,,,,,,,,,,,, +61000,0.4999995231628418,1.8213417530059808,,,,,,,,,,,,,, +61500,0.4999995827674866,3.1803879737854004,,,,,,,,,,,,,, +62000,0.4999995529651642,1.5683751106262207,,,,,,,,,,,,,, +62500,0.4999996423721313,1.9906294345855715,,,,,,,,,,,,,, +63000,0.4999995529651642,1.378468036651611,,,,,,,,,,,,,, +63303,,,0.68916015625,2.5399473571777342,0.6227,2.3859615625,50000.0,0.6045,1.758034765625,10000.0,51509.77036523819,54910.55043578148,51509.77036523819,3236.477827072144,8.006976842880249,0.0 +63500,0.4999996125698089,1.3786253929138184,,,,,,,,,,,,,, +64000,0.4999995827674866,1.413710117340088,,,,,,,,,,,,,, +64500,0.4999996423721313,1.6476373672485352,,,,,,,,,,,,,, +65000,0.4999995827674866,1.9774054288864136,,,,,,,,,,,,,, +65500,0.4999995529651642,2.519212484359741,,,,,,,,,,,,,, +66000,0.4999996125698089,3.122344493865967,,,,,,,,,,,,,, +66500,0.4999996423721313,1.479658603668213,,,,,,,,,,,,,, +66544,,,0.67556640625,2.814586181640625,0.61188,2.5821090625,50000.0,0.6037,1.736313671875,10000.0,54081.4677426815,57632.322525024414,54081.4677426815,3377.9268865585327,8.292526960372925,0.0 +67000,0.4999995529651642,2.235578775405884,,,,,,,,,,,,,, +67500,0.4999995827674866,2.69299054145813,,,,,,,,,,,,,, +68000,0.4999994933605194,2.7226898670196533,,,,,,,,,,,,,, +68500,0.4999996721744537,4.025160789489746,,,,,,,,,,,,,, +69000,0.4999996125698089,1.512830138206482,,,,,,,,,,,,,, +69500,0.4999997019767761,1.539435863494873,,,,,,,,,,,,,, +69665,,,0.69140625,2.551528015136719,0.61948,2.4735353125,50000.0,0.6067,1.7296421875,10000.0,56649.6544585228,60359.17927026749,56649.6544585228,3528.137953519821,8.600436449050903,0.0 +70000,0.4999996125698089,1.5503175258636477,,,,,,,,,,,,,, +70500,0.4999996423721313,1.4192150831222534,,,,,,,,,,,,,, +71000,0.4999996125698089,2.042428970336914,,,,,,,,,,,,,, +71500,0.4999995827674866,1.713430404663086,,,,,,,,,,,,,, +72000,0.4999996423721313,3.0458028316497803,,,,,,,,,,,,,, +72500,0.4999995231628418,1.3882684707641602,,,,,,,,,,,,,, +73000,0.4999996423721313,2.005265712738037,,,,,,,,,,,,,, +73022,,,0.713828125,2.382465362548828,0.6335,2.30428890625,50000.0,0.6071,1.7324955078125,10000.0,59218.12683725357,63080.01647400856,59218.12683725357,3671.800485849381,8.961181879043579,0.0 +73500,0.4999996125698089,1.3945821523666382,,,,,,,,,,,,,, +74000,0.4999995827674866,1.3234776258468628,,,,,,,,,,,,,, +74500,0.4999996423721313,3.5306968688964844,,,,,,,,,,,,,, +75000,0.4999996125698089,2.318986892700196,,,,,,,,,,,,,, +75500,0.4999996423721313,1.3640425205230713,,,,,,,,,,,,,, +76000,0.4999996423721313,1.347095012664795,,,,,,,,,,,,,, +76160,,,0.72046875,2.278394012451172,0.63344,2.3765715625,50000.0,0.6167,1.67845546875,10000.0,61793.17218565941,65822.94068861008,61793.17218565941,3831.300248384476,9.277946949005129,0.0 +76500,0.4999995827674866,1.420114278793335,,,,,,,,,,,,,, +77000,0.4999995529651642,2.175264596939087,,,,,,,,,,,,,, +77500,0.4999996125698089,1.877791166305542,,,,,,,,,,,,,, +78000,0.4999996423721313,3.727385044097901,,,,,,,,,,,,,, +78500,0.4999996125698089,3.046774387359619,,,,,,,,,,,,,, +79000,0.4999996721744537,1.4029536247253418,,,,,,,,,,,,,, +79490,,,0.68509765625,2.83866455078125,0.61306,2.6923934375,50000.0,0.6165,1.7130814453125,10000.0,64361.67498803139,68552.48768758774,64361.67498803139,3983.537580013275,9.606787919998167,0.0 +79500,0.4999996125698089,1.3300814628601074,,,,,,,,,,,,,, +80000,0.4999996125698089,1.5449459552764893,,,,,,,,,,,,,, +80500,0.4999995827674866,1.5910736322402954,,,,,,,,,,,,,, +81000,0.4999996125698089,2.157865524291992,,,,,,,,,,,,,, +81500,0.4999996125698089,1.3329180479049685,,,,,,,,,,,,,, +82000,0.4999996423721313,2.3120787143707275,,,,,,,,,,,,,, +82500,0.4999995827674866,1.222158670425415,,,,,,,,,,,,,, +82666,,,0.73498046875,2.2339772033691405,0.63742,2.43264578125,50000.0,0.6212,1.6881970703125,10000.0,66933.0193309784,71269.16496658325,66933.0193309784,4120.435905694962,9.923115968704224,0.0 +83000,0.4999996423721313,1.3570648431777954,,,,,,,,,,,,,, +83500,0.4999995827674866,1.339017391204834,,,,,,,,,,,,,, +84000,0.4999996721744537,1.392489314079285,,,,,,,,,,,,,, +84500,0.4999996423721313,3.597658157348633,,,,,,,,,,,,,, +85000,0.4999996125698089,2.027247905731201,,,,,,,,,,,,,, +85500,0.4999996423721313,1.3629871606826782,,,,,,,,,,,,,, +85838,,,0.7084765625,2.568540344238281,0.61908,2.64155625,50000.0,0.6217,1.6680759765625,10000.0,69504.26203966141,73995.53771471977,69504.26203966141,4266.311295747757,10.376159906387327,0.0 +86000,0.4999996721744537,1.4024487733840942,,,,,,,,,,,,,, +86500,0.4999996423721313,1.269100785255432,,,,,,,,,,,,,, +87000,0.4999996423721313,1.3212177753448486,,,,,,,,,,,,,, +87500,0.4999995827674866,1.938180446624756,,,,,,,,,,,,,, +88000,0.4999996125698089,2.253625869750977,,,,,,,,,,,,,, +88500,0.4999996125698089,1.6066205501556396,,,,,,,,,,,,,, +89000,0.4999995827674866,1.3541814088821411,,,,,,,,,,,,,, +89082,,,0.73923828125,2.213905029296875,0.62456,2.56997015625,50000.0,0.6228,1.673147265625,10000.0,72072.62209844589,76714.18280935287,72072.62209844589,4407.885825872421,10.789963960647585,0.0 +89500,0.4999995827674866,1.2925138473510742,,,,,,,,,,,,,, +90000,0.4999996423721313,1.944340705871582,,,,,,,,,,,,,, +90500,0.4999995827674866,1.3007527589797974,,,,,,,,,,,,,, +91000,0.4999996721744537,1.3713531494140625,,,,,,,,,,,,,, +91500,0.4999997019767761,3.699317455291748,,,,,,,,,,,,,, +92000,0.4999996721744537,1.3096656799316406,,,,,,,,,,,,,, +92224,,,0.72103515625,2.4955255126953126,0.63528,2.4236353125,50000.0,0.6227,1.670181640625,10000.0,74643.82851338387,79451.50010681152,74643.82851338387,4565.176446676254,11.278878450393677,0.0 +92500,0.4999995827674866,2.003863573074341,,,,,,,,,,,,,, +93000,0.4999996423721313,2.415379047393799,,,,,,,,,,,,,, +93500,0.4999996423721313,2.765591144561768,,,,,,,,,,,,,, +94000,0.4999996721744537,1.2334049940109253,,,,,,,,,,,,,, +94500,0.4999996721744537,2.8836469650268555,,,,,,,,,,,,,, +95000,0.4999996423721313,1.3924310207366943,,,,,,,,,,,,,, +95500,0.4999996125698089,1.2897558212280271,,,,,,,,,,,,,, +95603,,,0.75619140625,2.0707742309570314,0.62796,2.645846875,50000.0,0.6269,1.6623095703125,10000.0,77210.85782265663,82175.38922595978,77210.85782265663,4713.230769872665,11.575173377990724,0.0 +96000,0.4999997019767761,3.184514045715332,,,,,,,,,,,,,, +96500,0.4999996423721313,1.333088994026184,,,,,,,,,,,,,, +97000,0.4999996721744537,1.2995373010635376,,,,,,,,,,,,,, +97500,0.4999997019767761,3.7180075645446777,,,,,,,,,,,,,, +98000,0.4999997317790985,3.490739345550537,,,,,,,,,,,,,, +98500,0.4999995827674866,1.173065423965454,,,,,,,,,,,,,, +98746,,,0.7453125,2.3645455932617185,0.62078,2.8875203125,50000.0,0.6306,1.6648083984375,10000.0,79778.46589374542,84915.89196419716,79778.46589374542,4877.492128610611,11.848497152328491,0.0 +99000,0.4999997317790985,3.367938280105591,,,,,,,,,,,,,, +99500,0.4999996721744537,1.1905455589294434,,,,,,,,,,,,,, +100000,0.4999996125698089,1.5162746906280518,,,,,,,,,,,,,, +100500,0.4999996125698089,1.1451021432876587,,,,,,,,,,,,,, +101000,0.4999996721744537,1.3204824924468994,,,,,,,,,,,,,, +101500,0.4999996721744537,1.3307890892028809,,,,,,,,,,,,,, +102000,0.4999996423721313,1.3507238626480105,,,,,,,,,,,,,, +102025,,,0.72435546875,2.6406915283203123,0.62874,2.642935,50000.0,0.6328,1.66403203125,10000.0,82344.75841236115,87640.1489944458,82344.75841236115,5025.998027563095,12.233155250549316,0.0 +102500,0.4999997019767761,1.485764980316162,,,,,,,,,,,,,, +103000,0.4999996423721313,1.071098566055298,,,,,,,,,,,,,, +103500,0.4999996423721313,2.57316517829895,,,,,,,,,,,,,, +104000,0.4999996423721313,1.0278441905975342,,,,,,,,,,,,,, +104500,0.4999996721744537,1.1860451698303225,,,,,,,,,,,,,, +105000,0.4999996125698089,1.1821489334106443,,,,,,,,,,,,,, +105239,,,0.77421875,2.0628004455566407,0.66012,2.18563234375,50000.0,0.6329,1.6799451171875,10000.0,84911.81944584846,90360.26087784769,84911.81944584846,5170.570199251175,12.54542350769043,0.0 +105500,0.4999996423721313,2.763124227523804,,,,,,,,,,,,,, +106000,0.4999997019767761,1.8586326837539675,,,,,,,,,,,,,, +106500,0.4999996125698089,1.05022931098938,,,,,,,,,,,,,, +107000,0.4999997019767761,2.947037696838379,,,,,,,,,,,,,, +107500,0.4999996721744537,1.2741403579711914,,,,,,,,,,,,,, +108000,0.4999997019767761,2.7296605110168457,,,,,,,,,,,,,, +108407,,,0.709609375,3.0620135498046874,0.6479,2.51108796875,50000.0,0.635,1.6590978515625,10000.0,87478.88587450981,93080.1224527359,87478.88587450981,5313.689696073532,12.857575178146362,0.0 +108500,0.4999996721744537,3.1991119384765625,,,,,,,,,,,,,, +109000,0.4999996721744537,1.6077792644500732,,,,,,,,,,,,,, +109500,0.4999996721744537,1.3535395860671997,,,,,,,,,,,,,, +110000,0.4999996423721313,2.089114189147949,,,,,,,,,,,,,, +110500,0.4999996721744537,1.0785455703735352,,,,,,,,,,,,,, +111000,0.4999996423721313,1.2531802654266355,,,,,,,,,,,,,, +111500,0.4999996721744537,3.12183165550232,,,,,,,,,,,,,, +111719,,,0.74818359375,2.5472946166992188,0.6397,2.67240375,50000.0,0.6391,1.663360546875,10000.0,90045.97619056702,95798.57874584198,90045.97619056702,5456.271093606949,13.28238844871521,0.0 +112000,0.4999996721744537,1.283339262008667,,,,,,,,,,,,,, +112500,0.4999997019767761,1.825264573097229,,,,,,,,,,,,,, +113000,0.4999996721744537,1.734885811805725,,,,,,,,,,,,,, +113500,0.4999996721744537,2.9762887954711914,,,,,,,,,,,,,, +114000,0.4999997019767761,1.820704460144043,,,,,,,,,,,,,, +114500,0.4999996423721313,1.0848119258880615,,,,,,,,,,,,,, +114862,,,0.76982421875,2.211178894042969,0.63362,2.6727815625,50000.0,0.6416,1.6446095703125,10000.0,92615.26302433014,98538.27653622629,92615.26302433014,5617.700368642807,13.676774501800535,0.0 +115000,0.4999996125698089,1.326971173286438,,,,,,,,,,,,,, +115500,0.4999996721744537,2.407264471054077,,,,,,,,,,,,,, +116000,0.4999996721744537,1.1676881313323977,,,,,,,,,,,,,, +116500,0.4999996423721313,1.0845822095870972,,,,,,,,,,,,,, +117000,0.4999996721744537,2.1475563049316406,,,,,,,,,,,,,, +117500,0.4999996423721313,2.0687484741210938,,,,,,,,,,,,,, +118000,0.4999997019767761,1.1524797677993774,,,,,,,,,,,,,, +118220,,,0.7598046875,2.4921820068359377,0.64602,2.53122203125,50000.0,0.641,1.67543125,10000.0,95188.4490017891,101261.9952890873,95188.4490017891,5759.131661653519,13.992690801620483,0.0 +118500,0.4999996423721313,1.8224982023239136,,,,,,,,,,,,,, +119000,0.4999996423721313,1.304859280586243,,,,,,,,,,,,,, +119500,0.4999996423721313,1.2535521984100342,,,,,,,,,,,,,, +120000,0.4999997019767761,2.586952447891236,,,,,,,,,,,,,, +120500,0.4999996423721313,1.180744171142578,,,,,,,,,,,,,, +121000,0.4999997317790985,2.7898995876312256,,,,,,,,,,,,,, +121348,,,,,,,,,,,97758.78199744225,,,,,0.0 diff --git a/logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/trial_1/meta_data_0.json b/logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/trial_1/meta_data_0.json new file mode 100644 index 00000000..d98320e8 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/imagenet_vit_pytorch/trial_1/meta_data_0.json @@ -0,0 +1,74 @@ +{ + "workload.bn_init_scale": 0.0, + "workload.center_crop_size": 224, + "workload.eval_batch_size": 2048, + "workload.eval_num_workers": 0, + "workload.eval_period_time_sec": 2571, + "workload.max_allowed_runtime_sec": 64292, + "workload.num_eval_train_examples": 51200, + "workload.num_test_examples": 10000, + "workload.num_train_examples": 1281167, + "workload.num_validation_examples": 50000, + "workload.resize_size": 256, + "workload.step_hint": 167999, + "workload.target_metric_name": "accuracy", + "workload.test_target_value": 0.6518999999999999, + "workload.use_gelu": false, + "workload.use_glu": false, + "workload.use_map": false, + "workload.use_post_layer_norm": false, + "workload.use_silu": false, + "workload.validation_target_value": 0.77309, + "cpu.util.avg_percent_since_last": 7.0, + "cpu.freq.current": 2200.1959999999985, + "mem.total": 359053524992, + "mem.available": 347717935104, + "mem.used": 8179773440, + "mem.percent_used": 3.2, + "mem.read_bytes_since_boot": 38200440968704, + "mem.write_bytes_since_boot": 172476700160, + "net.bytes_sent_since_boot": 31341290, + "net.bytes_recv_since_boot": 31344313, + "gpu.count": 4, + "gpu.0.compute.util": 0.0, + "gpu.0.mem.util": 0.0314208984375, + "gpu.0.mem.total": 40960.0, + "gpu.0.mem.used": 1287.0, + "gpu.0.mem.free": 39040.0, + "gpu.0.temp.current": 33.0, + "gpu.1.compute.util": 0.0, + "gpu.1.mem.util": 0.0314208984375, + "gpu.1.mem.total": 40960.0, + "gpu.1.mem.used": 1287.0, + "gpu.1.mem.free": 39040.0, + "gpu.1.temp.current": 31.0, + "gpu.2.compute.util": 0.0, + "gpu.2.mem.util": 0.0314208984375, + "gpu.2.mem.total": 40960.0, + "gpu.2.mem.used": 1287.0, + "gpu.2.mem.free": 39040.0, + "gpu.2.temp.current": 31.0, + "gpu.3.compute.util": 0.0, + "gpu.3.mem.util": 0.0314208984375, + "gpu.3.mem.total": 40960.0, + "gpu.3.mem.used": 1287.0, + "gpu.3.mem.free": 39040.0, + "gpu.3.temp.current": 32.0, + "gpu.avg.compute.util": 0.0, + "gpu.avg.mem.util": 0.0314208984375, + "gpu.avg.mem.total": 40960.0, + "gpu.avg.mem.used": 1287.0, + "gpu.avg.mem.free": 39040.0, + "gpu.avg.temp.current": 31.75, + "os_platform": "Linux-6.1.0-44-cloud-amd64-x86_64-with-glibc2.31", + "python_version": "3.11.10", + "python_compiler": "GCC 9.4.0", + "git_branch": "main", + "git_commit_hash": "b21be29be0a1573fb4f78f849aea019cdb520862", + "cpu_model_name": "Intel(R) Xeon(R) CPU @ 2.20GHz", + "cpu_count": 24, + "gpu_model_name": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_driver": "550.90.12", + "rng_seed": -83011591 +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/librispeech_conformer_pytorch_09-16-2026-18-33-54.log b/logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/librispeech_conformer_pytorch_09-16-2026-18-33-54.log new file mode 100644 index 00000000..095dbb4c --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/librispeech_conformer_pytorch_09-16-2026-18-33-54.log @@ -0,0 +1,1186 @@ +torchrun --redirects 1:0,2:0,3:0 --standalone --nnodes=1 --nproc_per_node=4 submission_runner.py --framework=pytorch --workload=librispeech_conformer --submission_path=submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py --data_dir=/data/librispeech --experiment_dir=/experiment_runs --experiment_name=submissions_a100/ademamix_golden/study_2 --overwrite=True --save_checkpoints=False --rng_seed=-582177137 --librispeech_tokenizer_vocab_path=/data/librispeech/spm_model.vocab --torch_compile=true --tuning_ruleset=self 2>&1 | tee -a /logs/librispeech_conformer_pytorch_09-16-2026-18-33-54.log +W0916 18:33:56.632000 9 site-packages/torch/distributed/run.py:803] +W0916 18:33:56.632000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0916 18:33:56.632000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0916 18:33:56.632000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-16 18:33:58.148801: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 18:33:58.148801: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 18:33:58.148804: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 18:33:58.148805: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789583638.173707 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789583638.173707 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789583638.173708 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789583638.173707 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789583638.181807 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789583638.181807 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789583638.181810 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789583638.181812 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789583638.208335 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789583638.208335 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789583638.208335 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789583638.208334 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789583638.208365 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789583638.208366 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789583638.208366 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789583638.208368 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789583638.208368 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789583638.208369 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789583638.208370 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789583638.208371 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789583638.208371 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789583638.208370 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789583638.208373 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789583638.208376 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789583643.943995 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789583644.036007 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789583644.130960 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +W0000 00:00:1789583644.455571 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W916 18:34:06.511371783 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0916 18:34:06.954484 139718034855104 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/librispeech_conformer_pytorch. +I0916 18:34:06.954484 139910084355264 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/librispeech_conformer_pytorch. +I0916 18:34:06.954483 139813834572992 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/librispeech_conformer_pytorch. +I0916 18:34:06.954509 139755322442944 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/librispeech_conformer_pytorch. +I0916 18:34:06.978447 139813834572992 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/librispeech_conformer_pytorch/trial_1. +I0916 18:34:07.257793 139813834572992 submission_runner.py:242] Initializing dataset. +I0916 18:34:07.257975 139813834572992 input_pipeline.py:19] Loading split = train-clean-100 +I0916 18:34:07.294783 139813834572992 input_pipeline.py:19] Loading split = train-clean-360 +I0916 18:34:07.465233 139813834572992 input_pipeline.py:19] Loading split = train-other-500 +I0916 18:34:07.965387 139813834572992 submission_runner.py:251] Initializing model. +W0916 18:34:09.367436 139755322442944 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0916 18:34:09.367655 139910084355264 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0916 18:34:09.367858 139718034855104 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0916 18:34:09.367856 139813834572992 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +I0916 18:34:09.368030 139813834572992 submission_runner.py:294] Initializing optimizer. +I0916 18:34:09.368728 139813834572992 submission_runner.py:299] Initializing metrics bundle. +I0916 18:34:09.368889 139813834572992 submission_runner.py:321] Initializing checkpoint and logger. +I0916 18:34:09.369473 139813834572992 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_2/librispeech_conformer_pytorch/trial_1/meta_data_0.json. +I0916 18:34:09.369557 139755322442944 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 18:34:09.369576 139910084355264 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 18:34:09.369585 139718034855104 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 18:34:09.369681 139755322442944 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 18:34:09.369682 139813834572992 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 18:34:09.369698 139910084355264 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 18:34:09.369699 139718034855104 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 18:34:09.369736 139813834572992 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 18:34:09.639550 139813834572992 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_2/librispeech_conformer_pytorch/trial_1/flags_0.json. +I0916 18:34:09.658327 139813834572992 submission_runner.py:359] Starting training loop. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/algorithmic-efficiency/algoperf/workloads/librispeech_conformer/librispeech_pytorch/preprocessor.py:516: UserWarning: Specified kernel cache directory could not be created! This disables kernel caching. Specified directory is /root/.cache/torch/kernels. This warning will appear only once per process. (Triggered internally at /pytorch/aten/src/ATen/native/cuda/jit_utils.cpp:1487.) + spectrum = torch.abs(spectrum) +/algorithmic-efficiency/algoperf/workloads/librispeech_conformer/librispeech_pytorch/preprocessor.py:516: UserWarning: Specified kernel cache directory could not be created! This disables kernel caching. Specified directory is /root/.cache/torch/kernels. This warning will appear only once per process. (Triggered internally at /pytorch/aten/src/ATen/native/cuda/jit_utils.cpp:1487.) + spectrum = torch.abs(spectrum) +/algorithmic-efficiency/algoperf/workloads/librispeech_conformer/librispeech_pytorch/preprocessor.py:516: UserWarning: Specified kernel cache directory could not be created! This disables kernel caching. Specified directory is /root/.cache/torch/kernels. This warning will appear only once per process. (Triggered internally at /pytorch/aten/src/ATen/native/cuda/jit_utils.cpp:1487.) + spectrum = torch.abs(spectrum) +I0916 18:34:23.658409 139796569351936 logging_writer.py:48] [0] global_step=0, grad_norm=0.5, loss=33.3706 +I0916 18:34:23.708832 139813834572992 submission.py:307] 0) loss = 33.371, grad_norm = 0.500 +I0916 18:34:24.623784 139813834572992 spec.py:333] Evaluating on the training split. +I0916 18:34:24.624964 139813834572992 input_pipeline.py:19] Loading split = train-clean-100 +I0916 18:34:24.704132 139813834572992 input_pipeline.py:19] Loading split = train-clean-360 +I0916 18:34:24.804580 139813834572992 input_pipeline.py:19] Loading split = train-other-500 +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 18:34:57.421189 139813834572992 spec.py:346] Evaluating on the validation split. +I0916 18:34:57.422510 139813834572992 input_pipeline.py:19] Loading split = dev-clean +I0916 18:34:57.426314 139813834572992 input_pipeline.py:19] Loading split = dev-other +I0916 18:35:17.478151 139813834572992 spec.py:363] Evaluating on the test split. +I0916 18:35:17.479333 139813834572992 input_pipeline.py:19] Loading split = test-clean +I0916 18:35:27.841156 139813834572992 submission_runner.py:516] Time since start: 78.18s, Step: 1, {'train/ctc_loss': 32.10654239086722, 'train/wer': 1.7203084470202628, 'validation/ctc_loss': 30.97378880349608, 'validation/wer': 1.8542557813933278, 'validation/num_examples': 5348, 'test/ctc_loss': 31.06509452222724, 'test/wer': 1.8702496293136717, 'test/num_examples': 2472, 'score': 14.052122116088867, 'total_duration': 78.18270134925842, 'accumulated_submission_time': 14.052122116088867, 'accumulated_eval_time': 63.21826100349426, 'accumulated_logging_time': 0} +I0916 18:35:27.867756 139795539912448 logging_writer.py:48] [1] accumulated_eval_time=63.2183, accumulated_logging_time=0, accumulated_submission_time=14.0521, global_step=1, preemption_count=0, score=14.0521, test/ctc_loss=31.0651, test/num_examples=2472, test/wer=1.87025, total_duration=78.1827, train/ctc_loss=32.1065, train/wer=1.72031, validation/ctc_loss=30.9738, validation/num_examples=5348, validation/wer=1.85426 +I0916 18:35:30.218269 139795531519744 logging_writer.py:48] [1] global_step=1, grad_norm=0.5, loss=32.6383 +I0916 18:35:30.221257 139813834572992 submission.py:307] 1) loss = 32.638, grad_norm = 0.500 +I0916 18:35:30.888065 139795539912448 logging_writer.py:48] [2] global_step=2, grad_norm=0.5, loss=33.1394 +I0916 18:35:30.891171 139813834572992 submission.py:307] 2) loss = 33.139, grad_norm = 0.500 +I0916 18:35:31.336345 139795531519744 logging_writer.py:48] [3] global_step=3, grad_norm=0.5, loss=33.2361 +I0916 18:35:31.339812 139813834572992 submission.py:307] 3) loss = 33.236, grad_norm = 0.500 +I0916 18:35:31.784933 139795539912448 logging_writer.py:48] [4] global_step=4, grad_norm=0.5, loss=32.7417 +I0916 18:35:31.788036 139813834572992 submission.py:307] 4) loss = 32.742, grad_norm = 0.500 +I0916 18:35:32.232688 139795531519744 logging_writer.py:48] [5] global_step=5, grad_norm=0.5, loss=33.0579 +I0916 18:35:32.235754 139813834572992 submission.py:307] 5) loss = 33.058, grad_norm = 0.500 +I0916 18:35:32.681736 139795539912448 logging_writer.py:48] [6] global_step=6, grad_norm=0.5, loss=33.0201 +I0916 18:35:32.684875 139813834572992 submission.py:307] 6) loss = 33.020, grad_norm = 0.500 +I0916 18:35:33.130729 139795531519744 logging_writer.py:48] [7] global_step=7, grad_norm=0.5, loss=31.9478 +I0916 18:35:33.133773 139813834572992 submission.py:307] 7) loss = 31.948, grad_norm = 0.500 +I0916 18:35:33.578534 139795539912448 logging_writer.py:48] [8] global_step=8, grad_norm=0.5, loss=32.4804 +I0916 18:35:33.581507 139813834572992 submission.py:307] 8) loss = 32.480, grad_norm = 0.500 +I0916 18:35:34.026540 139795531519744 logging_writer.py:48] [9] global_step=9, grad_norm=0.5, loss=32.0381 +I0916 18:35:34.029615 139813834572992 submission.py:307] 9) loss = 32.038, grad_norm = 0.500 +I0916 18:35:35.640310 139795539912448 logging_writer.py:48] [10] global_step=10, grad_norm=0.5, loss=31.9755 +I0916 18:35:35.643492 139813834572992 submission.py:307] 10) loss = 31.975, grad_norm = 0.500 +I0916 18:35:36.622175 139795531519744 logging_writer.py:48] [11] global_step=11, grad_norm=0.5, loss=32.026 +I0916 18:35:36.625104 139813834572992 submission.py:307] 11) loss = 32.026, grad_norm = 0.500 +I0916 18:35:37.564547 139795539912448 logging_writer.py:48] [12] global_step=12, grad_norm=0.5, loss=32.1755 +I0916 18:35:37.568455 139813834572992 submission.py:307] 12) loss = 32.176, grad_norm = 0.500 +I0916 18:35:39.003542 139795531519744 logging_writer.py:48] [13] global_step=13, grad_norm=0.5, loss=31.5741 +I0916 18:35:39.006456 139813834572992 submission.py:307] 13) loss = 31.574, grad_norm = 0.500 +I0916 18:35:41.014039 139795539912448 logging_writer.py:48] [14] global_step=14, grad_norm=0.5, loss=31.3315 +I0916 18:35:41.016951 139813834572992 submission.py:307] 14) loss = 31.331, grad_norm = 0.500 +I0916 18:35:41.905930 139795531519744 logging_writer.py:48] [15] global_step=15, grad_norm=0.5, loss=30.4017 +I0916 18:35:41.908793 139813834572992 submission.py:307] 15) loss = 30.402, grad_norm = 0.500 +I0916 18:35:42.924305 139795539912448 logging_writer.py:48] [16] global_step=16, grad_norm=0.5, loss=31.0127 +I0916 18:35:42.927228 139813834572992 submission.py:307] 16) loss = 31.013, grad_norm = 0.500 +I0916 18:35:44.307738 139795531519744 logging_writer.py:48] [17] global_step=17, grad_norm=0.5, loss=30.0301 +I0916 18:35:44.310795 139813834572992 submission.py:307] 17) loss = 30.030, grad_norm = 0.500 +I0916 18:35:46.223735 139795539912448 logging_writer.py:48] [18] global_step=18, grad_norm=0.5, loss=29.6761 +I0916 18:35:46.226672 139813834572992 submission.py:307] 18) loss = 29.676, grad_norm = 0.500 +I0916 18:35:47.561869 139795531519744 logging_writer.py:48] [19] global_step=19, grad_norm=0.5, loss=29.099 +I0916 18:35:47.564872 139813834572992 submission.py:307] 19) loss = 29.099, grad_norm = 0.500 +I0916 18:35:48.336605 139795539912448 logging_writer.py:48] [20] global_step=20, grad_norm=0.5, loss=28.1335 +I0916 18:35:48.339571 139813834572992 submission.py:307] 20) loss = 28.133, grad_norm = 0.500 +I0916 18:35:49.774887 139795531519744 logging_writer.py:48] [21] global_step=21, grad_norm=0.5, loss=27.8411 +I0916 18:35:49.778032 139813834572992 submission.py:307] 21) loss = 27.841, grad_norm = 0.500 +I0916 18:35:51.638880 139795539912448 logging_writer.py:48] [22] global_step=22, grad_norm=0.5, loss=28.0482 +I0916 18:35:51.641796 139813834572992 submission.py:307] 22) loss = 28.048, grad_norm = 0.500 +I0916 18:35:52.758047 139795531519744 logging_writer.py:48] [23] global_step=23, grad_norm=0.5, loss=26.3505 +I0916 18:35:52.760940 139813834572992 submission.py:307] 23) loss = 26.350, grad_norm = 0.500 +I0916 18:35:53.815766 139795539912448 logging_writer.py:48] [24] global_step=24, grad_norm=0.5, loss=25.9661 +I0916 18:35:53.818656 139813834572992 submission.py:307] 24) loss = 25.966, grad_norm = 0.500 +I0916 18:35:55.670562 139795531519744 logging_writer.py:48] [25] global_step=25, grad_norm=0.5, loss=25.3123 +I0916 18:35:55.673705 139813834572992 submission.py:307] 25) loss = 25.312, grad_norm = 0.500 +I0916 18:35:57.207908 139795539912448 logging_writer.py:48] [26] global_step=26, grad_norm=0.5, loss=24.3226 +I0916 18:35:57.210854 139813834572992 submission.py:307] 26) loss = 24.323, grad_norm = 0.500 +I0916 18:35:58.000041 139795531519744 logging_writer.py:48] [27] global_step=27, grad_norm=0.5, loss=23.3153 +I0916 18:35:58.003130 139813834572992 submission.py:307] 27) loss = 23.315, grad_norm = 0.500 +I0916 18:35:59.404110 139795539912448 logging_writer.py:48] [28] global_step=28, grad_norm=0.5, loss=21.9084 +I0916 18:35:59.407066 139813834572992 submission.py:307] 28) loss = 21.908, grad_norm = 0.500 +I0916 18:36:00.438750 139795531519744 logging_writer.py:48] [29] global_step=29, grad_norm=0.5, loss=22.1079 +I0916 18:36:00.441866 139813834572992 submission.py:307] 29) loss = 22.108, grad_norm = 0.500 +I0916 18:36:02.620793 139795539912448 logging_writer.py:48] [30] global_step=30, grad_norm=0.5, loss=21.1877 +I0916 18:36:02.623721 139813834572992 submission.py:307] 30) loss = 21.188, grad_norm = 0.500 +I0916 18:36:03.675746 139795531519744 logging_writer.py:48] [31] global_step=31, grad_norm=0.5, loss=20.7397 +I0916 18:36:03.678659 139813834572992 submission.py:307] 31) loss = 20.740, grad_norm = 0.500 +I0916 18:36:04.880854 139795539912448 logging_writer.py:48] [32] global_step=32, grad_norm=0.5, loss=20.1848 +I0916 18:36:04.883851 139813834572992 submission.py:307] 32) loss = 20.185, grad_norm = 0.500 +I0916 18:36:06.168416 139795531519744 logging_writer.py:48] [33] global_step=33, grad_norm=0.5, loss=19.5695 +I0916 18:36:06.171377 139813834572992 submission.py:307] 33) loss = 19.570, grad_norm = 0.500 +I0916 18:36:08.203041 139795539912448 logging_writer.py:48] [34] global_step=34, grad_norm=0.5, loss=19.7156 +I0916 18:36:08.205960 139813834572992 submission.py:307] 34) loss = 19.716, grad_norm = 0.500 +I0916 18:36:08.967458 139795531519744 logging_writer.py:48] [35] global_step=35, grad_norm=0.5, loss=19.3151 +I0916 18:36:08.970383 139813834572992 submission.py:307] 35) loss = 19.315, grad_norm = 0.500 +I0916 18:36:10.100602 139795539912448 logging_writer.py:48] [36] global_step=36, grad_norm=0.5, loss=18.2117 +I0916 18:36:10.103509 139813834572992 submission.py:307] 36) loss = 18.212, grad_norm = 0.500 +I0916 18:36:11.627454 139795531519744 logging_writer.py:48] [37] global_step=37, grad_norm=0.5, loss=17.6531 +I0916 18:36:11.630367 139813834572992 submission.py:307] 37) loss = 17.653, grad_norm = 0.500 +I0916 18:36:13.410929 139795539912448 logging_writer.py:48] [38] global_step=38, grad_norm=0.5, loss=17.0001 +I0916 18:36:13.413812 139813834572992 submission.py:307] 38) loss = 17.000, grad_norm = 0.500 +I0916 18:36:14.621042 139795531519744 logging_writer.py:48] [39] global_step=39, grad_norm=0.5, loss=17.0253 +I0916 18:36:14.624109 139813834572992 submission.py:307] 39) loss = 17.025, grad_norm = 0.500 +I0916 18:36:15.449419 139795539912448 logging_writer.py:48] [40] global_step=40, grad_norm=0.5, loss=16.5716 +I0916 18:36:15.452437 139813834572992 submission.py:307] 40) loss = 16.572, grad_norm = 0.500 +I0916 18:36:16.616197 139795531519744 logging_writer.py:48] [41] global_step=41, grad_norm=0.5, loss=15.9312 +I0916 18:36:16.619085 139813834572992 submission.py:307] 41) loss = 15.931, grad_norm = 0.500 +I0916 18:36:18.963992 139795539912448 logging_writer.py:48] [42] global_step=42, grad_norm=0.5, loss=15.4232 +I0916 18:36:18.966978 139813834572992 submission.py:307] 42) loss = 15.423, grad_norm = 0.500 +I0916 18:36:19.829961 139795531519744 logging_writer.py:48] [43] global_step=43, grad_norm=0.5, loss=15.0516 +I0916 18:36:19.832901 139813834572992 submission.py:307] 43) loss = 15.052, grad_norm = 0.500 +I0916 18:36:21.364803 139795539912448 logging_writer.py:48] [44] global_step=44, grad_norm=0.5, loss=15.1935 +I0916 18:36:21.367692 139813834572992 submission.py:307] 44) loss = 15.193, grad_norm = 0.500 +I0916 18:36:22.058045 139795531519744 logging_writer.py:48] [45] global_step=45, grad_norm=0.5, loss=14.7661 +I0916 18:36:22.060945 139813834572992 submission.py:307] 45) loss = 14.766, grad_norm = 0.500 +I0916 18:36:24.382755 139795539912448 logging_writer.py:48] [46] global_step=46, grad_norm=0.5, loss=13.5831 +I0916 18:36:24.385857 139813834572992 submission.py:307] 46) loss = 13.583, grad_norm = 0.500 +I0916 18:36:25.117429 139795531519744 logging_writer.py:48] [47] global_step=47, grad_norm=0.5, loss=13.5047 +I0916 18:36:25.120383 139813834572992 submission.py:307] 47) loss = 13.505, grad_norm = 0.500 +I0916 18:36:26.994199 139795539912448 logging_writer.py:48] [48] global_step=48, grad_norm=0.5, loss=14.2677 +I0916 18:36:26.997263 139813834572992 submission.py:307] 48) loss = 14.268, grad_norm = 0.500 +I0916 18:36:27.469827 139795531519744 logging_writer.py:48] [49] global_step=49, grad_norm=0.5, loss=14.8659 +I0916 18:36:27.472756 139813834572992 submission.py:307] 49) loss = 14.866, grad_norm = 0.500 +I0916 18:36:29.713412 139795539912448 logging_writer.py:48] [50] global_step=50, grad_norm=0.5, loss=13.9269 +I0916 18:36:29.716507 139813834572992 submission.py:307] 50) loss = 13.927, grad_norm = 0.500 +I0916 18:36:30.440063 139795531519744 logging_writer.py:48] [51] global_step=51, grad_norm=0.5, loss=12.4105 +I0916 18:36:30.442914 139813834572992 submission.py:307] 51) loss = 12.410, grad_norm = 0.500 +I0916 18:36:32.194255 139795539912448 logging_writer.py:48] [52] global_step=52, grad_norm=0.5, loss=12.6304 +I0916 18:36:32.197184 139813834572992 submission.py:307] 52) loss = 12.630, grad_norm = 0.500 +I0916 18:36:33.091384 139795531519744 logging_writer.py:48] [53] global_step=53, grad_norm=0.5, loss=12.6022 +I0916 18:36:33.094443 139813834572992 submission.py:307] 53) loss = 12.602, grad_norm = 0.500 +I0916 18:36:34.672444 139795539912448 logging_writer.py:48] [54] global_step=54, grad_norm=0.5, loss=13.2105 +I0916 18:36:34.675431 139813834572992 submission.py:307] 54) loss = 13.210, grad_norm = 0.500 +I0916 18:36:35.868531 139795531519744 logging_writer.py:48] [55] global_step=55, grad_norm=0.5, loss=12.4894 +I0916 18:36:35.871553 139813834572992 submission.py:307] 55) loss = 12.489, grad_norm = 0.500 +I0916 18:36:37.932793 139795539912448 logging_writer.py:48] [56] global_step=56, grad_norm=0.5, loss=12.5996 +I0916 18:36:37.935797 139813834572992 submission.py:307] 56) loss = 12.600, grad_norm = 0.500 +I0916 18:36:38.483635 139795531519744 logging_writer.py:48] [57] global_step=57, grad_norm=0.5, loss=11.9669 +I0916 18:36:38.487030 139813834572992 submission.py:307] 57) loss = 11.967, grad_norm = 0.500 +I0916 18:36:39.999971 139795539912448 logging_writer.py:48] [58] global_step=58, grad_norm=0.5, loss=12.1164 +I0916 18:36:40.003030 139813834572992 submission.py:307] 58) loss = 12.116, grad_norm = 0.500 +I0916 18:36:41.423616 139795531519744 logging_writer.py:48] [59] global_step=59, grad_norm=0.5, loss=11.6123 +I0916 18:36:41.426626 139813834572992 submission.py:307] 59) loss = 11.612, grad_norm = 0.500 +I0916 18:36:43.313383 139795539912448 logging_writer.py:48] [60] global_step=60, grad_norm=0.5, loss=11.6197 +I0916 18:36:43.316386 139813834572992 submission.py:307] 60) loss = 11.620, grad_norm = 0.500 +I0916 18:36:44.142190 139795531519744 logging_writer.py:48] [61] global_step=61, grad_norm=0.5, loss=11.1781 +I0916 18:36:44.145123 139813834572992 submission.py:307] 61) loss = 11.178, grad_norm = 0.500 +I0916 18:36:45.629083 139795539912448 logging_writer.py:48] [62] global_step=62, grad_norm=0.5, loss=11.4171 +I0916 18:36:45.632069 139813834572992 submission.py:307] 62) loss = 11.417, grad_norm = 0.500 +I0916 18:36:47.059247 139795531519744 logging_writer.py:48] [63] global_step=63, grad_norm=0.5, loss=11.3564 +I0916 18:36:47.062162 139813834572992 submission.py:307] 63) loss = 11.356, grad_norm = 0.500 +I0916 18:36:49.328899 139795539912448 logging_writer.py:48] [64] global_step=64, grad_norm=0.5, loss=10.41 +I0916 18:36:49.331797 139813834572992 submission.py:307] 64) loss = 10.410, grad_norm = 0.500 +I0916 18:36:50.156647 139795531519744 logging_writer.py:48] [65] global_step=65, grad_norm=0.5, loss=11.1622 +I0916 18:36:50.159675 139813834572992 submission.py:307] 65) loss = 11.162, grad_norm = 0.500 +I0916 18:36:50.605543 139795539912448 logging_writer.py:48] [66] global_step=66, grad_norm=0.5, loss=11.2813 +I0916 18:36:50.608436 139813834572992 submission.py:307] 66) loss = 11.281, grad_norm = 0.500 +I0916 18:36:52.796123 139795531519744 logging_writer.py:48] [67] global_step=67, grad_norm=0.5, loss=11.06 +I0916 18:36:52.799149 139813834572992 submission.py:307] 67) loss = 11.060, grad_norm = 0.500 +I0916 18:36:54.510628 139795539912448 logging_writer.py:48] [68] global_step=68, grad_norm=0.5, loss=10.6369 +I0916 18:36:54.513622 139813834572992 submission.py:307] 68) loss = 10.637, grad_norm = 0.500 +I0916 18:36:55.014965 139795531519744 logging_writer.py:48] [69] global_step=69, grad_norm=0.5, loss=11.1488 +I0916 18:36:55.017982 139813834572992 submission.py:307] 69) loss = 11.149, grad_norm = 0.500 +I0916 18:36:55.536055 139795539912448 logging_writer.py:48] [70] global_step=70, grad_norm=0.5, loss=10.6698 +I0916 18:36:55.539094 139813834572992 submission.py:307] 70) loss = 10.670, grad_norm = 0.500 +I0916 18:36:57.937498 139795531519744 logging_writer.py:48] [71] global_step=71, grad_norm=0.5, loss=11.1246 +I0916 18:36:57.940423 139813834572992 submission.py:307] 71) loss = 11.125, grad_norm = 0.500 +I0916 18:36:59.608913 139795539912448 logging_writer.py:48] [72] global_step=72, grad_norm=0.5, loss=10.4306 +I0916 18:36:59.611874 139813834572992 submission.py:307] 72) loss = 10.431, grad_norm = 0.500 +I0916 18:37:00.057614 139795531519744 logging_writer.py:48] [73] global_step=73, grad_norm=0.5, loss=10.6261 +I0916 18:37:00.060461 139813834572992 submission.py:307] 73) loss = 10.626, grad_norm = 0.500 +I0916 18:37:00.946335 139795539912448 logging_writer.py:48] [74] global_step=74, grad_norm=0.5, loss=10.9696 +I0916 18:37:00.949366 139813834572992 submission.py:307] 74) loss = 10.970, grad_norm = 0.500 +I0916 18:37:03.228069 139795531519744 logging_writer.py:48] [75] global_step=75, grad_norm=0.5, loss=10.6285 +I0916 18:37:03.230976 139813834572992 submission.py:307] 75) loss = 10.628, grad_norm = 0.500 +I0916 18:37:05.237534 139795539912448 logging_writer.py:48] [76] global_step=76, grad_norm=0.5, loss=9.97252 +I0916 18:37:05.240635 139813834572992 submission.py:307] 76) loss = 9.973, grad_norm = 0.500 +I0916 18:37:05.687885 139795531519744 logging_writer.py:48] [77] global_step=77, grad_norm=0.5, loss=10.0869 +I0916 18:37:05.691067 139813834572992 submission.py:307] 77) loss = 10.087, grad_norm = 0.500 +I0916 18:37:06.137177 139795539912448 logging_writer.py:48] [78] global_step=78, grad_norm=0.5, loss=9.98147 +I0916 18:37:06.140147 139813834572992 submission.py:307] 78) loss = 9.981, grad_norm = 0.500 +I0916 18:37:08.634868 139795531519744 logging_writer.py:48] [79] global_step=79, grad_norm=0.5, loss=9.25859 +I0916 18:37:08.637925 139813834572992 submission.py:307] 79) loss = 9.259, grad_norm = 0.500 +I0916 18:37:10.869304 139795539912448 logging_writer.py:48] [80] global_step=80, grad_norm=0.5, loss=9.08051 +I0916 18:37:10.872311 139813834572992 submission.py:307] 80) loss = 9.081, grad_norm = 0.500 +I0916 18:37:11.319795 139795531519744 logging_writer.py:48] [81] global_step=81, grad_norm=0.5, loss=8.53504 +I0916 18:37:11.322840 139813834572992 submission.py:307] 81) loss = 8.535, grad_norm = 0.500 +I0916 18:37:11.768667 139795539912448 logging_writer.py:48] [82] global_step=82, grad_norm=0.5, loss=7.44135 +I0916 18:37:11.771697 139813834572992 submission.py:307] 82) loss = 7.441, grad_norm = 0.500 +I0916 18:37:14.358553 139795531519744 logging_writer.py:48] [83] global_step=83, grad_norm=0.5, loss=7.22112 +I0916 18:37:14.361464 139813834572992 submission.py:307] 83) loss = 7.221, grad_norm = 0.500 +I0916 18:37:15.840099 139795539912448 logging_writer.py:48] [84] global_step=84, grad_norm=0.5, loss=7.40768 +I0916 18:37:15.843127 139813834572992 submission.py:307] 84) loss = 7.408, grad_norm = 0.500 +I0916 18:37:16.290844 139795531519744 logging_writer.py:48] [85] global_step=85, grad_norm=0.5, loss=7.36434 +I0916 18:37:16.294229 139813834572992 submission.py:307] 85) loss = 7.364, grad_norm = 0.500 +I0916 18:37:16.743333 139795539912448 logging_writer.py:48] [86] global_step=86, grad_norm=0.5, loss=7.18682 +I0916 18:37:16.746286 139813834572992 submission.py:307] 86) loss = 7.187, grad_norm = 0.500 +I0916 18:37:19.488262 139795531519744 logging_writer.py:48] [87] global_step=87, grad_norm=0.5, loss=6.88257 +I0916 18:37:19.491283 139813834572992 submission.py:307] 87) loss = 6.883, grad_norm = 0.500 +I0916 18:37:21.099202 139795539912448 logging_writer.py:48] [88] global_step=88, grad_norm=0.5, loss=6.59268 +I0916 18:37:21.102640 139813834572992 submission.py:307] 88) loss = 6.593, grad_norm = 0.500 +I0916 18:37:21.548796 139795531519744 logging_writer.py:48] [89] global_step=89, grad_norm=0.5, loss=6.5932 +I0916 18:37:21.552094 139813834572992 submission.py:307] 89) loss = 6.593, grad_norm = 0.500 +I0916 18:37:21.997763 139795539912448 logging_writer.py:48] [90] global_step=90, grad_norm=0.5, loss=6.73077 +I0916 18:37:22.000766 139813834572992 submission.py:307] 90) loss = 6.731, grad_norm = 0.500 +I0916 18:37:24.776907 139795531519744 logging_writer.py:48] [91] global_step=91, grad_norm=0.5, loss=6.77348 +I0916 18:37:24.779844 139813834572992 submission.py:307] 91) loss = 6.773, grad_norm = 0.500 +I0916 18:37:26.165335 139795539912448 logging_writer.py:48] [92] global_step=92, grad_norm=0.5, loss=6.60172 +I0916 18:37:26.168277 139813834572992 submission.py:307] 92) loss = 6.602, grad_norm = 0.500 +I0916 18:37:26.614543 139795531519744 logging_writer.py:48] [93] global_step=93, grad_norm=0.5, loss=6.44242 +I0916 18:37:26.617568 139813834572992 submission.py:307] 93) loss = 6.442, grad_norm = 0.500 +I0916 18:37:27.065034 139795539912448 logging_writer.py:48] [94] global_step=94, grad_norm=0.5, loss=6.47037 +I0916 18:37:27.068045 139813834572992 submission.py:307] 94) loss = 6.470, grad_norm = 0.500 +I0916 18:37:29.645380 139795531519744 logging_writer.py:48] [95] global_step=95, grad_norm=0.5, loss=6.47552 +I0916 18:37:29.648515 139813834572992 submission.py:307] 95) loss = 6.476, grad_norm = 0.500 +I0916 18:37:31.585922 139795539912448 logging_writer.py:48] [96] global_step=96, grad_norm=0.5, loss=6.43656 +I0916 18:37:31.589045 139813834572992 submission.py:307] 96) loss = 6.437, grad_norm = 0.500 +I0916 18:37:32.036751 139795531519744 logging_writer.py:48] [97] global_step=97, grad_norm=0.5, loss=6.359 +I0916 18:37:32.039989 139813834572992 submission.py:307] 97) loss = 6.359, grad_norm = 0.500 +I0916 18:37:32.560898 139795539912448 logging_writer.py:48] [98] global_step=98, grad_norm=0.5, loss=6.37443 +I0916 18:37:32.564054 139813834572992 submission.py:307] 98) loss = 6.374, grad_norm = 0.500 +I0916 18:37:35.512280 139795531519744 logging_writer.py:48] [99] global_step=99, grad_norm=0.5, loss=6.4224 +I0916 18:37:35.515243 139813834572992 submission.py:307] 99) loss = 6.422, grad_norm = 0.500 +I0916 18:37:36.990484 139795539912448 logging_writer.py:48] [100] global_step=100, grad_norm=0.5, loss=6.38508 +I0916 18:37:36.993442 139813834572992 submission.py:307] 100) loss = 6.385, grad_norm = 0.500 +I0916 18:46:44.196820 139795531519744 logging_writer.py:48] [500] global_step=500, grad_norm=0.5, loss=5.76955 +I0916 18:46:44.285425 139813834572992 submission.py:307] 500) loss = 5.770, grad_norm = 0.500 +I0916 18:58:20.452933 139795539912448 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.5, loss=3.5657 +I0916 18:58:20.456754 139813834572992 submission.py:307] 1000) loss = 3.566, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 19:04:36.960429 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +I0916 19:04:55.466211 139813834572992 spec.py:346] Evaluating on the validation split. +I0916 19:05:15.241543 139813834572992 spec.py:363] Evaluating on the test split. +I0916 19:05:25.489505 139813834572992 submission_runner.py:516] Time since start: 1875.83s, Step: 1409, {'train/ctc_loss': 4.387652035647058, 'train/wer': 0.7925199969611131, 'validation/ctc_loss': 4.561761838833634, 'validation/wer': 0.7672765895814223, 'validation/num_examples': 5348, 'test/ctc_loss': 4.362626217820504, 'test/wer': 0.756240732841793, 'test/num_examples': 2472, 'score': 1760.6268198490143, 'total_duration': 1875.831004858017, 'accumulated_submission_time': 1760.6268198490143, 'accumulated_eval_time': 111.74717354774475, 'accumulated_logging_time': 0.036690473556518555} +I0916 19:05:25.519756 139795539912448 logging_writer.py:48] [1409] accumulated_eval_time=111.747, accumulated_logging_time=0.0366905, accumulated_submission_time=1760.63, global_step=1409, preemption_count=0, score=1760.63, test/ctc_loss=4.36263, test/num_examples=2472, test/wer=0.756241, total_duration=1875.83, train/ctc_loss=4.38765, train/wer=0.79252, validation/ctc_loss=4.56176, validation/num_examples=5348, validation/wer=0.767277 +I0916 19:07:09.629621 139795531519744 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.499999, loss=2.76864 +I0916 19:07:09.633294 139813834572992 submission.py:307] 1500) loss = 2.769, grad_norm = 0.500 +I0916 19:18:24.455431 139795539912448 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.499999, loss=2.29685 +I0916 19:18:24.459571 139813834572992 submission.py:307] 2000) loss = 2.297, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 19:24:24.507773 139795539912448 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.499999, loss=2.23736 +I0916 19:24:24.514941 139813834572992 submission.py:307] 2500) loss = 2.237, grad_norm = 0.500 +I0916 19:33:59.370858 139795531519744 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.499999, loss=1.96977 +I0916 19:33:59.375034 139813834572992 submission.py:307] 3000) loss = 1.970, grad_norm = 0.500 +I0916 19:34:33.760014 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 19:34:53.419596 139813834572992 spec.py:346] Evaluating on the validation split. +I0916 19:35:13.485912 139813834572992 spec.py:363] Evaluating on the test split. +I0916 19:35:23.755020 139813834572992 submission_runner.py:516] Time since start: 3674.10s, Step: 3026, {'train/ctc_loss': 1.3805324669399563, 'train/wer': 0.41440649453543016, 'validation/ctc_loss': 1.5873540748380048, 'validation/wer': 0.42313522908318446, 'validation/num_examples': 5348, 'test/ctc_loss': 1.2777354587392735, 'test/wer': 0.37539861475026914, 'test/num_examples': 2472, 'score': 3506.2168979644775, 'total_duration': 3674.096529006958, 'accumulated_submission_time': 3506.2168979644775, 'accumulated_eval_time': 161.74285888671875, 'accumulated_logging_time': 0.07685184478759766} +I0916 19:35:23.786921 139795539912448 logging_writer.py:48] [3026] accumulated_eval_time=161.743, accumulated_logging_time=0.0768518, accumulated_submission_time=3506.22, global_step=3026, preemption_count=0, score=3506.22, test/ctc_loss=1.27774, test/num_examples=2472, test/wer=0.375399, total_duration=3674.1, train/ctc_loss=1.38053, train/wer=0.414406, validation/ctc_loss=1.58735, validation/num_examples=5348, validation/wer=0.423135 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 19:40:17.170530 139795539912448 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.499999, loss=1.87246 +I0916 19:40:17.177764 139813834572992 submission.py:307] 3500) loss = 1.872, grad_norm = 0.500 +I0916 19:48:08.101334 139795531519744 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.499999, loss=1.83772 +I0916 19:48:08.105583 139813834572992 submission.py:307] 4000) loss = 1.838, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 19:53:28.670294 139795539912448 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.499999, loss=1.74528 +I0916 19:53:28.677682 139813834572992 submission.py:307] 4500) loss = 1.745, grad_norm = 0.500 +I0916 20:00:24.532485 139795531519744 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.499999, loss=1.76297 +I0916 20:00:24.536530 139813834572992 submission.py:307] 5000) loss = 1.763, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 20:04:31.890009 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +I0916 20:04:48.323177 139813834572992 spec.py:346] Evaluating on the validation split. +I0916 20:05:08.282835 139813834572992 spec.py:363] Evaluating on the test split. +I0916 20:05:18.622757 139813834572992 submission_runner.py:516] Time since start: 5468.96s, Step: 5304, {'train/ctc_loss': 0.5142548732515267, 'train/wer': 0.17535462724796233, 'validation/ctc_loss': 0.724086608948036, 'validation/wer': 0.2171003717472119, 'validation/num_examples': 5348, 'test/ctc_loss': 0.4732933878798632, 'test/wer': 0.15832876322791622, 'test/num_examples': 2472, 'score': 5251.343508243561, 'total_duration': 5468.964254617691, 'accumulated_submission_time': 5251.343508243561, 'accumulated_eval_time': 208.47549104690552, 'accumulated_logging_time': 0.11817240715026855} +I0916 20:05:18.655356 139795539912448 logging_writer.py:48] [5304] accumulated_eval_time=208.475, accumulated_logging_time=0.118172, accumulated_submission_time=5251.34, global_step=5304, preemption_count=0, score=5251.34, test/ctc_loss=0.473293, test/num_examples=2472, test/wer=0.158329, total_duration=5468.96, train/ctc_loss=0.514255, train/wer=0.175355, validation/ctc_loss=0.724087, validation/num_examples=5348, validation/wer=0.2171 +I0916 20:06:52.785379 139795531519744 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.499999, loss=1.71536 +I0916 20:06:52.789771 139813834572992 submission.py:307] 5500) loss = 1.715, grad_norm = 0.500 +I0916 20:13:43.006325 139795539912448 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.499999, loss=1.58801 +I0916 20:13:43.010550 139813834572992 submission.py:307] 6000) loss = 1.588, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 20:19:28.807082 139795539912448 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.499999, loss=1.52421 +I0916 20:19:28.823300 139813834572992 submission.py:307] 6500) loss = 1.524, grad_norm = 0.500 +I0916 20:25:41.484709 139795531519744 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.499999, loss=1.598 +I0916 20:25:41.488939 139813834572992 submission.py:307] 7000) loss = 1.598, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 20:31:47.289643 139795539912448 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.499999, loss=1.52675 +I0916 20:31:47.297033 139813834572992 submission.py:307] 7500) loss = 1.527, grad_norm = 0.500 +I0916 20:34:27.074877 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 20:34:41.888352 139813834572992 spec.py:346] Evaluating on the validation split. +I0916 20:35:01.891685 139813834572992 spec.py:363] Evaluating on the test split. +I0916 20:35:12.255431 139813834572992 submission_runner.py:516] Time since start: 7262.60s, Step: 7782, {'train/ctc_loss': 0.4007701530663972, 'train/wer': 0.14094465969893313, 'validation/ctc_loss': 0.6215434749849307, 'validation/wer': 0.1870612658716748, 'validation/num_examples': 5348, 'test/ctc_loss': 0.3864171428398606, 'test/wer': 0.1295472548900128, 'test/num_examples': 2472, 'score': 6996.924051761627, 'total_duration': 7262.596936225891, 'accumulated_submission_time': 6996.924051761627, 'accumulated_eval_time': 253.65583658218384, 'accumulated_logging_time': 0.16046833992004395} +I0916 20:35:12.308894 139795539912448 logging_writer.py:48] [7782] accumulated_eval_time=253.656, accumulated_logging_time=0.160468, accumulated_submission_time=6996.92, global_step=7782, preemption_count=0, score=6996.92, test/ctc_loss=0.386417, test/num_examples=2472, test/wer=0.129547, total_duration=7262.6, train/ctc_loss=0.40077, train/wer=0.140945, validation/ctc_loss=0.621543, validation/num_examples=5348, validation/wer=0.187061 +I0916 20:38:28.640335 139795531519744 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.499999, loss=1.54628 +I0916 20:38:28.644354 139813834572992 submission.py:307] 8000) loss = 1.546, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 20:44:49.752389 139795539912448 logging_writer.py:48] [8500] global_step=8500, grad_norm=0.499999, loss=1.48423 +I0916 20:44:49.759767 139813834572992 submission.py:307] 8500) loss = 1.484, grad_norm = 0.500 +I0916 20:50:22.161697 139795531519744 logging_writer.py:48] [9000] global_step=9000, grad_norm=0.499999, loss=1.53066 +I0916 20:50:22.165992 139813834572992 submission.py:307] 9000) loss = 1.531, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 20:57:02.463531 139795539912448 logging_writer.py:48] [9500] global_step=9500, grad_norm=0.499999, loss=1.51082 +I0916 20:57:02.470980 139813834572992 submission.py:307] 9500) loss = 1.511, grad_norm = 0.500 +I0916 21:02:17.325306 139795531519744 logging_writer.py:48] [10000] global_step=10000, grad_norm=0.499999, loss=1.50837 +I0916 21:02:17.329489 139813834572992 submission.py:307] 10000) loss = 1.508, grad_norm = 0.500 +I0916 21:04:23.315392 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 21:04:39.415779 139813834572992 spec.py:346] Evaluating on the validation split. +I0916 21:04:59.072346 139813834572992 spec.py:363] Evaluating on the test split. +I0916 21:05:09.534810 139813834572992 submission_runner.py:516] Time since start: 9059.88s, Step: 10121, {'train/ctc_loss': 0.3454691138252321, 'train/wer': 0.12270048513658711, 'validation/ctc_loss': 0.5673220839298272, 'validation/wer': 0.1700477960701009, 'validation/num_examples': 5348, 'test/ctc_loss': 0.3434555253564746, 'test/wer': 0.1161213007535596, 'test/num_examples': 2472, 'score': 8744.24173116684, 'total_duration': 9059.876303434372, 'accumulated_submission_time': 8744.24173116684, 'accumulated_eval_time': 299.8750274181366, 'accumulated_logging_time': 0.22359585762023926} +I0916 21:05:09.841861 139795539912448 logging_writer.py:48] [10121] accumulated_eval_time=299.875, accumulated_logging_time=0.223596, accumulated_submission_time=8744.24, global_step=10121, preemption_count=0, score=8744.24, test/ctc_loss=0.343456, test/num_examples=2472, test/wer=0.116121, total_duration=9059.88, train/ctc_loss=0.345469, train/wer=0.1227, validation/ctc_loss=0.567322, validation/num_examples=5348, validation/wer=0.170048 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 21:09:55.028584 139795539912448 logging_writer.py:48] [10500] global_step=10500, grad_norm=0.499999, loss=1.51652 +I0916 21:09:55.036281 139813834572992 submission.py:307] 10500) loss = 1.517, grad_norm = 0.500 +I0916 21:15:04.831578 139795531519744 logging_writer.py:48] [11000] global_step=11000, grad_norm=0.499999, loss=1.45466 +I0916 21:15:04.835781 139813834572992 submission.py:307] 11000) loss = 1.455, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 21:22:11.514145 139795539912448 logging_writer.py:48] [11500] global_step=11500, grad_norm=0.499999, loss=1.38937 +I0916 21:22:11.521509 139813834572992 submission.py:307] 11500) loss = 1.389, grad_norm = 0.500 +I0916 21:26:55.640038 139795531519744 logging_writer.py:48] [12000] global_step=12000, grad_norm=0.499999, loss=1.43209 +I0916 21:26:55.644263 139813834572992 submission.py:307] 12000) loss = 1.432, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 21:34:17.939959 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +I0916 21:34:35.743090 139813834572992 spec.py:346] Evaluating on the validation split. +I0916 21:34:55.571876 139813834572992 spec.py:363] Evaluating on the test split. +I0916 21:35:05.877568 139813834572992 submission_runner.py:516] Time since start: 10856.22s, Step: 12489, {'train/ctc_loss': 0.3102659863434054, 'train/wer': 0.11110387566611316, 'validation/ctc_loss': 0.5330505187142104, 'validation/wer': 0.15943610293052674, 'validation/num_examples': 5348, 'test/ctc_loss': 0.3173984007032712, 'test/wer': 0.10767168362683566, 'test/num_examples': 2472, 'score': 10489.384288549423, 'total_duration': 10856.218958854675, 'accumulated_submission_time': 10489.384288549423, 'accumulated_eval_time': 347.81264901161194, 'accumulated_logging_time': 0.540952205657959} +I0916 21:35:05.989670 139795539912448 logging_writer.py:48] [12489] accumulated_eval_time=347.813, accumulated_logging_time=0.540952, accumulated_submission_time=10489.4, global_step=12489, preemption_count=0, score=10489.4, test/ctc_loss=0.317398, test/num_examples=2472, test/wer=0.107672, total_duration=10856.2, train/ctc_loss=0.310266, train/wer=0.111104, validation/ctc_loss=0.533051, validation/num_examples=5348, validation/wer=0.159436 +I0916 21:35:13.787492 139795531519744 logging_writer.py:48] [12500] global_step=12500, grad_norm=0.499999, loss=1.36452 +I0916 21:35:13.790977 139813834572992 submission.py:307] 12500) loss = 1.365, grad_norm = 0.500 +I0916 21:40:02.186139 139795539912448 logging_writer.py:48] [13000] global_step=13000, grad_norm=0.499999, loss=1.45055 +I0916 21:40:02.190377 139813834572992 submission.py:307] 13000) loss = 1.451, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 21:47:39.138627 139795539912448 logging_writer.py:48] [13500] global_step=13500, grad_norm=0.499999, loss=1.31791 +I0916 21:47:39.181739 139813834572992 submission.py:307] 13500) loss = 1.318, grad_norm = 0.500 +I0916 21:52:04.877923 139795531519744 logging_writer.py:48] [14000] global_step=14000, grad_norm=0.499999, loss=1.41986 +I0916 21:52:04.910431 139813834572992 submission.py:307] 14000) loss = 1.420, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 21:59:56.661956 139795539912448 logging_writer.py:48] [14500] global_step=14500, grad_norm=0.499999, loss=1.34361 +I0916 21:59:56.700772 139813834572992 submission.py:307] 14500) loss = 1.344, grad_norm = 0.500 +I0916 22:04:12.775587 139795531519744 logging_writer.py:48] [15000] global_step=15000, grad_norm=0.499999, loss=1.41012 +I0916 22:04:12.779926 139813834572992 submission.py:307] 15000) loss = 1.410, grad_norm = 0.500 +I0916 22:04:14.983795 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 22:04:30.784650 139813834572992 spec.py:346] Evaluating on the validation split. +I0916 22:04:50.415762 139813834572992 spec.py:363] Evaluating on the test split. +I0916 22:05:00.584242 139813834572992 submission_runner.py:516] Time since start: 12650.93s, Step: 15002, {'train/ctc_loss': 0.27723881361502667, 'train/wer': 0.10048405127036325, 'validation/ctc_loss': 0.502230916512834, 'validation/wer': 0.14948100226910635, 'validation/num_examples': 5348, 'test/ctc_loss': 0.29786130355184204, 'test/wer': 0.09918144334084862, 'test/num_examples': 2472, 'score': 12234.653105258942, 'total_duration': 12650.925751447678, 'accumulated_submission_time': 12234.653105258942, 'accumulated_eval_time': 393.4130597114563, 'accumulated_logging_time': 0.6633203029632568} +I0916 22:05:00.935068 139795539912448 logging_writer.py:48] [15002] accumulated_eval_time=393.413, accumulated_logging_time=0.66332, accumulated_submission_time=12234.7, global_step=15002, preemption_count=0, score=12234.7, test/ctc_loss=0.297861, test/num_examples=2472, test/wer=0.0991814, total_duration=12650.9, train/ctc_loss=0.277239, train/wer=0.100484, validation/ctc_loss=0.502231, validation/num_examples=5348, validation/wer=0.149481 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 22:12:56.631918 139795539912448 logging_writer.py:48] [15500] global_step=15500, grad_norm=0.493755, loss=1.30995 +I0916 22:12:56.639521 139813834572992 submission.py:307] 15500) loss = 1.310, grad_norm = 0.494 +I0916 22:17:03.361969 139795531519744 logging_writer.py:48] [16000] global_step=16000, grad_norm=0.463725, loss=1.34561 +I0916 22:17:03.366389 139813834572992 submission.py:307] 16000) loss = 1.346, grad_norm = 0.464 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 22:25:06.414862 139795539912448 logging_writer.py:48] [16500] global_step=16500, grad_norm=0.499999, loss=1.35523 +I0916 22:25:06.422513 139813834572992 submission.py:307] 16500) loss = 1.355, grad_norm = 0.500 +I0916 22:29:05.633742 139795531519744 logging_writer.py:48] [17000] global_step=17000, grad_norm=0.499999, loss=1.35285 +I0916 22:29:05.638044 139813834572992 submission.py:307] 17000) loss = 1.353, grad_norm = 0.500 +I0916 22:34:08.781001 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 22:34:24.180215 139813834572992 spec.py:346] Evaluating on the validation split. +I0916 22:34:44.989048 139813834572992 spec.py:363] Evaluating on the test split. +I0916 22:34:56.346176 139813834572992 submission_runner.py:516] Time since start: 14446.69s, Step: 17340, {'train/ctc_loss': 0.25481339725955643, 'train/wer': 0.092962806194988, 'validation/ctc_loss': 0.475153822819344, 'validation/wer': 0.14224882923767682, 'validation/num_examples': 5348, 'test/ctc_loss': 0.27807923091812375, 'test/wer': 0.09402230211443544, 'test/num_examples': 2472, 'score': 13979.592563390732, 'total_duration': 14446.687247037888, 'accumulated_submission_time': 13979.592563390732, 'accumulated_eval_time': 440.97758173942566, 'accumulated_logging_time': 1.0245919227600098} +I0916 22:34:56.508944 139795539912448 logging_writer.py:48] [17340] accumulated_eval_time=440.978, accumulated_logging_time=1.02459, accumulated_submission_time=13979.6, global_step=17340, preemption_count=0, score=13979.6, test/ctc_loss=0.278079, test/num_examples=2472, test/wer=0.0940223, total_duration=14446.7, train/ctc_loss=0.254813, train/wer=0.0929628, validation/ctc_loss=0.475154, validation/num_examples=5348, validation/wer=0.142249 +I0916 22:37:55.081931 139795531519744 logging_writer.py:48] [17500] global_step=17500, grad_norm=0.446921, loss=1.32308 +I0916 22:37:55.086084 139813834572992 submission.py:307] 17500) loss = 1.323, grad_norm = 0.447 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 22:42:05.071826 139795539912448 logging_writer.py:48] [18000] global_step=18000, grad_norm=0.499999, loss=1.39373 +I0916 22:42:05.079419 139813834572992 submission.py:307] 18000) loss = 1.394, grad_norm = 0.500 +I0916 22:49:51.698618 139795531519744 logging_writer.py:48] [18500] global_step=18500, grad_norm=0.499999, loss=1.27736 +I0916 22:49:51.704083 139813834572992 submission.py:307] 18500) loss = 1.277, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 22:54:07.314026 139795539912448 logging_writer.py:48] [19000] global_step=19000, grad_norm=0.470878, loss=1.25872 +I0916 22:54:07.321434 139813834572992 submission.py:307] 19000) loss = 1.259, grad_norm = 0.471 +I0916 23:01:43.318054 139795531519744 logging_writer.py:48] [19500] global_step=19500, grad_norm=0.499999, loss=1.39189 +I0916 23:01:43.322229 139813834572992 submission.py:307] 19500) loss = 1.392, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 23:04:04.454011 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +I0916 23:04:18.329405 139813834572992 spec.py:346] Evaluating on the validation split. +I0916 23:04:37.995057 139813834572992 spec.py:363] Evaluating on the test split. +I0916 23:04:48.387059 139813834572992 submission_runner.py:516] Time since start: 16238.73s, Step: 19706, {'train/ctc_loss': 0.23629783483327194, 'train/wer': 0.08583770173325085, 'validation/ctc_loss': 0.46606361921087003, 'validation/wer': 0.1356925602278762, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2691817314826763, 'test/wer': 0.08819287876018118, 'test/num_examples': 2472, 'score': 15724.528203010559, 'total_duration': 16238.72852897644, 'accumulated_submission_time': 15724.528203010559, 'accumulated_eval_time': 484.9104630947113, 'accumulated_logging_time': 1.1978044509887695} +I0916 23:04:48.633661 139795539912448 logging_writer.py:48] [19706] accumulated_eval_time=484.91, accumulated_logging_time=1.1978, accumulated_submission_time=15724.5, global_step=19706, preemption_count=0, score=15724.5, test/ctc_loss=0.269182, test/num_examples=2472, test/wer=0.0881929, total_duration=16238.7, train/ctc_loss=0.236298, train/wer=0.0858377, validation/ctc_loss=0.466064, validation/num_examples=5348, validation/wer=0.135693 +I0916 23:07:14.729961 139795531519744 logging_writer.py:48] [20000] global_step=20000, grad_norm=0.439291, loss=1.35552 +I0916 23:07:14.734160 139813834572992 submission.py:307] 20000) loss = 1.356, grad_norm = 0.439 +I0916 23:14:40.489406 139795539912448 logging_writer.py:48] [20500] global_step=20500, grad_norm=0.499999, loss=1.29525 +I0916 23:14:40.493479 139813834572992 submission.py:307] 20500) loss = 1.295, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 23:19:30.410614 139795539912448 logging_writer.py:48] [21000] global_step=21000, grad_norm=0.499999, loss=1.3193 +I0916 23:19:30.417825 139813834572992 submission.py:307] 21000) loss = 1.319, grad_norm = 0.500 +I0916 23:26:30.836284 139795531519744 logging_writer.py:48] [21500] global_step=21500, grad_norm=0.485028, loss=1.35517 +I0916 23:26:30.840574 139813834572992 submission.py:307] 21500) loss = 1.355, grad_norm = 0.485 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 23:31:43.606088 139795539912448 logging_writer.py:48] [22000] global_step=22000, grad_norm=0.477887, loss=1.21538 +I0916 23:31:43.613900 139813834572992 submission.py:307] 22000) loss = 1.215, grad_norm = 0.478 +I0916 23:33:57.373574 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 23:34:18.738834 139813834572992 spec.py:346] Evaluating on the validation split. +I0916 23:34:38.881450 139813834572992 spec.py:363] Evaluating on the test split. +I0916 23:34:49.167146 139813834572992 submission_runner.py:516] Time since start: 18039.51s, Step: 22219, {'train/ctc_loss': 0.22142080195318706, 'train/wer': 0.08291819967657561, 'validation/ctc_loss': 0.4489882953774864, 'validation/wer': 0.13268961521749625, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2557915026775921, 'test/wer': 0.08429305547092397, 'test/num_examples': 2472, 'score': 17469.457185268402, 'total_duration': 18039.508671998978, 'accumulated_submission_time': 17469.457185268402, 'accumulated_eval_time': 536.7038400173187, 'accumulated_logging_time': 1.4543979167938232} +I0916 23:34:49.438987 139795539912448 logging_writer.py:48] [22219] accumulated_eval_time=536.704, accumulated_logging_time=1.4544, accumulated_submission_time=17469.5, global_step=22219, preemption_count=0, score=17469.5, test/ctc_loss=0.255792, test/num_examples=2472, test/wer=0.0842931, total_duration=18039.5, train/ctc_loss=0.221421, train/wer=0.0829182, validation/ctc_loss=0.448988, validation/num_examples=5348, validation/wer=0.13269 +I0916 23:39:16.626271 139795531519744 logging_writer.py:48] [22500] global_step=22500, grad_norm=0.499999, loss=1.2746 +I0916 23:39:16.630574 139813834572992 submission.py:307] 22500) loss = 1.275, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 23:44:47.966470 139795539912448 logging_writer.py:48] [23000] global_step=23000, grad_norm=0.467911, loss=1.28403 +I0916 23:44:47.973826 139813834572992 submission.py:307] 23000) loss = 1.284, grad_norm = 0.468 +I0916 23:51:07.968050 139795531519744 logging_writer.py:48] [23500] global_step=23500, grad_norm=0.499999, loss=1.16627 +I0916 23:51:07.972340 139813834572992 submission.py:307] 23500) loss = 1.166, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 23:56:56.086783 139795539912448 logging_writer.py:48] [24000] global_step=24000, grad_norm=0.499999, loss=1.26764 +I0916 23:56:56.094259 139813834572992 submission.py:307] 24000) loss = 1.268, grad_norm = 0.500 +I0917 00:02:55.398390 139795531519744 logging_writer.py:48] [24500] global_step=24500, grad_norm=0.499999, loss=1.26962 +I0917 00:02:55.402513 139813834572992 submission.py:307] 24500) loss = 1.270, grad_norm = 0.500 +I0917 00:03:57.368783 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 00:04:13.734472 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 00:04:33.333059 139813834572992 spec.py:363] Evaluating on the test split. +I0917 00:04:43.672952 139813834572992 submission_runner.py:516] Time since start: 19834.01s, Step: 24558, {'train/ctc_loss': 0.206900756080581, 'train/wer': 0.07622179533096735, 'validation/ctc_loss': 0.4338568253811282, 'validation/wer': 0.12752377733790374, 'validation/num_examples': 5348, 'test/ctc_loss': 0.24766864999354798, 'test/wer': 0.08151036906140191, 'test/num_examples': 2472, 'score': 19214.567471265793, 'total_duration': 19834.014476537704, 'accumulated_submission_time': 19214.567471265793, 'accumulated_eval_time': 583.007815361023, 'accumulated_logging_time': 1.7365632057189941} +I0917 00:04:43.745744 139795539912448 logging_writer.py:48] [24558] accumulated_eval_time=583.008, accumulated_logging_time=1.73656, accumulated_submission_time=19214.6, global_step=24558, preemption_count=0, score=19214.6, test/ctc_loss=0.247669, test/num_examples=2472, test/wer=0.0815104, total_duration=19834, train/ctc_loss=0.206901, train/wer=0.0762218, validation/ctc_loss=0.433857, validation/num_examples=5348, validation/wer=0.127524 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 00:09:46.667254 139795539912448 logging_writer.py:48] [25000] global_step=25000, grad_norm=0.499999, loss=1.294 +I0917 00:09:46.674876 139813834572992 submission.py:307] 25000) loss = 1.294, grad_norm = 0.500 +I0917 00:15:40.404408 139795531519744 logging_writer.py:48] [25500] global_step=25500, grad_norm=0.499999, loss=1.23502 +I0917 00:15:40.408493 139813834572992 submission.py:307] 25500) loss = 1.235, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 00:22:04.210022 139795539912448 logging_writer.py:48] [26000] global_step=26000, grad_norm=0.499999, loss=1.20904 +I0917 00:22:04.217320 139813834572992 submission.py:307] 26000) loss = 1.209, grad_norm = 0.500 +I0917 00:27:29.511564 139795531519744 logging_writer.py:48] [26500] global_step=26500, grad_norm=0.499999, loss=1.27691 +I0917 00:27:29.515803 139813834572992 submission.py:307] 26500) loss = 1.277, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 00:33:52.208766 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +I0917 00:34:06.259385 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 00:34:25.898285 139813834572992 spec.py:363] Evaluating on the test split. +I0917 00:34:36.342787 139813834572992 submission_runner.py:516] Time since start: 21626.68s, Step: 26938, {'train/ctc_loss': 0.19763641386815184, 'train/wer': 0.07440931635897936, 'validation/ctc_loss': 0.4246268185214487, 'validation/wer': 0.12472360353401246, 'validation/num_examples': 5348, 'test/ctc_loss': 0.24187261073295052, 'test/wer': 0.0804947900798245, 'test/num_examples': 2472, 'score': 20960.067337989807, 'total_duration': 21626.68429684639, 'accumulated_submission_time': 20960.067337989807, 'accumulated_eval_time': 627.1416909694672, 'accumulated_logging_time': 1.8191895484924316} +I0917 00:34:36.731073 139795539912448 logging_writer.py:48] [26938] accumulated_eval_time=627.142, accumulated_logging_time=1.81919, accumulated_submission_time=20960.1, global_step=26938, preemption_count=0, score=20960.1, test/ctc_loss=0.241873, test/num_examples=2472, test/wer=0.0804948, total_duration=21626.7, train/ctc_loss=0.197636, train/wer=0.0744093, validation/ctc_loss=0.424627, validation/num_examples=5348, validation/wer=0.124724 +I0917 00:35:06.266345 139795531519744 logging_writer.py:48] [27000] global_step=27000, grad_norm=0.5, loss=1.17745 +I0917 00:35:06.270322 139813834572992 submission.py:307] 27000) loss = 1.177, grad_norm = 0.500 +I0917 00:40:31.377521 139795539912448 logging_writer.py:48] [27500] global_step=27500, grad_norm=0.499999, loss=1.21374 +I0917 00:40:31.381764 139813834572992 submission.py:307] 27500) loss = 1.214, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 00:47:27.863088 139795539912448 logging_writer.py:48] [28000] global_step=28000, grad_norm=0.499999, loss=1.20161 +I0917 00:47:27.870745 139813834572992 submission.py:307] 28000) loss = 1.202, grad_norm = 0.500 +I0917 00:52:21.708529 139795531519744 logging_writer.py:48] [28500] global_step=28500, grad_norm=0.499999, loss=1.11147 +I0917 00:52:21.712613 139813834572992 submission.py:307] 28500) loss = 1.111, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 00:59:41.186328 139795539912448 logging_writer.py:48] [29000] global_step=29000, grad_norm=0.499999, loss=1.13505 +I0917 00:59:41.193825 139813834572992 submission.py:307] 29000) loss = 1.135, grad_norm = 0.500 +I0917 01:03:45.987461 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 01:04:04.195026 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 01:04:24.069927 139813834572992 spec.py:363] Evaluating on the test split. +I0917 01:04:34.259344 139813834572992 submission_runner.py:516] Time since start: 23424.60s, Step: 29454, {'train/ctc_loss': 0.18498094590758396, 'train/wer': 0.06965020241157381, 'validation/ctc_loss': 0.4099803186454566, 'validation/wer': 0.11964466760005793, 'validation/num_examples': 5348, 'test/ctc_loss': 0.23302987793567326, 'test/wer': 0.07480754778299108, 'test/num_examples': 2472, 'score': 22705.55812382698, 'total_duration': 23424.600848913193, 'accumulated_submission_time': 22705.55812382698, 'accumulated_eval_time': 675.4134092330933, 'accumulated_logging_time': 2.2177481651306152} +I0917 01:04:34.633581 139795539912448 logging_writer.py:48] [29454] accumulated_eval_time=675.413, accumulated_logging_time=2.21775, accumulated_submission_time=22705.6, global_step=29454, preemption_count=0, score=22705.6, test/ctc_loss=0.23303, test/num_examples=2472, test/wer=0.0748075, total_duration=23424.6, train/ctc_loss=0.184981, train/wer=0.0696502, validation/ctc_loss=0.40998, validation/num_examples=5348, validation/wer=0.119645 +I0917 01:05:12.808038 139795531519744 logging_writer.py:48] [29500] global_step=29500, grad_norm=0.499999, loss=1.20683 +I0917 01:05:12.811731 139813834572992 submission.py:307] 29500) loss = 1.207, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 01:12:39.561921 139795539912448 logging_writer.py:48] [30000] global_step=30000, grad_norm=0.425477, loss=1.27877 +I0917 01:12:39.569286 139813834572992 submission.py:307] 30000) loss = 1.279, grad_norm = 0.425 +I0917 01:17:12.065992 139795531519744 logging_writer.py:48] [30500] global_step=30500, grad_norm=0.424522, loss=1.22136 +I0917 01:17:12.070396 139813834572992 submission.py:307] 30500) loss = 1.221, grad_norm = 0.425 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 01:24:50.064125 139795539912448 logging_writer.py:48] [31000] global_step=31000, grad_norm=0.499999, loss=1.15816 +I0917 01:24:50.071739 139813834572992 submission.py:307] 31000) loss = 1.158, grad_norm = 0.500 +I0917 01:29:11.252565 139795531519744 logging_writer.py:48] [31500] global_step=31500, grad_norm=0.477724, loss=1.1858 +I0917 01:29:11.256771 139813834572992 submission.py:307] 31500) loss = 1.186, grad_norm = 0.478 +I0917 01:33:44.738660 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 01:33:59.880570 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 01:34:19.678800 139813834572992 spec.py:363] Evaluating on the test split. +I0917 01:34:30.068380 139813834572992 submission_runner.py:516] Time since start: 25220.41s, Step: 31784, {'train/ctc_loss': 0.17568948841842372, 'train/wer': 0.06572135577768372, 'validation/ctc_loss': 0.40605113600688164, 'validation/wer': 0.11707623231786801, 'validation/num_examples': 5348, 'test/ctc_loss': 0.22631442451932382, 'test/wer': 0.07328417931062499, 'test/num_examples': 2472, 'score': 24452.785871744156, 'total_duration': 25220.40988755226, 'accumulated_submission_time': 24452.785871744156, 'accumulated_eval_time': 720.7431962490082, 'accumulated_logging_time': 2.6020264625549316} +I0917 01:34:30.147614 139795539912448 logging_writer.py:48] [31784] accumulated_eval_time=720.743, accumulated_logging_time=2.60203, accumulated_submission_time=24452.8, global_step=31784, preemption_count=0, score=24452.8, test/ctc_loss=0.226314, test/num_examples=2472, test/wer=0.0732842, total_duration=25220.4, train/ctc_loss=0.175689, train/wer=0.0657214, validation/ctc_loss=0.406051, validation/num_examples=5348, validation/wer=0.117076 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 01:37:44.493504 139795539912448 logging_writer.py:48] [32000] global_step=32000, grad_norm=0.499999, loss=1.19732 +I0917 01:37:44.501828 139813834572992 submission.py:307] 32000) loss = 1.197, grad_norm = 0.500 +I0917 01:42:07.764791 139795531519744 logging_writer.py:48] [32500] global_step=32500, grad_norm=0.409768, loss=1.18171 +I0917 01:42:07.768927 139813834572992 submission.py:307] 32500) loss = 1.182, grad_norm = 0.410 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 01:50:02.675217 139795539912448 logging_writer.py:48] [33000] global_step=33000, grad_norm=0.499999, loss=1.147 +I0917 01:50:02.682471 139813834572992 submission.py:307] 33000) loss = 1.147, grad_norm = 0.500 +I0917 01:54:05.597284 139795531519744 logging_writer.py:48] [33500] global_step=33500, grad_norm=0.5, loss=1.21147 +I0917 01:54:05.601526 139813834572992 submission.py:307] 33500) loss = 1.211, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 02:02:17.757389 139795539912448 logging_writer.py:48] [34000] global_step=34000, grad_norm=0.499999, loss=1.15534 +I0917 02:02:17.764554 139813834572992 submission.py:307] 34000) loss = 1.155, grad_norm = 0.500 +I0917 02:03:39.065387 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 02:03:53.903871 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 02:04:14.436197 139813834572992 spec.py:363] Evaluating on the test split. +I0917 02:04:25.426002 139813834572992 submission_runner.py:516] Time since start: 27015.77s, Step: 34184, {'train/ctc_loss': 0.16585316436071523, 'train/wer': 0.062329740934891846, 'validation/ctc_loss': 0.39722547939270647, 'validation/wer': 0.11447882972046541, 'validation/num_examples': 5348, 'test/ctc_loss': 0.22110098332956965, 'test/wer': 0.0696484065565779, 'test/num_examples': 2472, 'score': 26198.306342363358, 'total_duration': 27015.766971111298, 'accumulated_submission_time': 26198.306342363358, 'accumulated_eval_time': 767.1031429767609, 'accumulated_logging_time': 2.691215991973877} +I0917 02:04:25.790897 139795539912448 logging_writer.py:48] [34184] accumulated_eval_time=767.103, accumulated_logging_time=2.69122, accumulated_submission_time=26198.3, global_step=34184, preemption_count=0, score=26198.3, test/ctc_loss=0.221101, test/num_examples=2472, test/wer=0.0696484, total_duration=27015.8, train/ctc_loss=0.165853, train/wer=0.0623297, validation/ctc_loss=0.397225, validation/num_examples=5348, validation/wer=0.114479 +I0917 02:07:16.416006 139795531519744 logging_writer.py:48] [34500] global_step=34500, grad_norm=0.499999, loss=1.14768 +I0917 02:07:16.420374 139813834572992 submission.py:307] 34500) loss = 1.148, grad_norm = 0.500 +I0917 02:15:19.976825 139795539912448 logging_writer.py:48] [35000] global_step=35000, grad_norm=0.499999, loss=1.12759 +I0917 02:15:19.980823 139813834572992 submission.py:307] 35000) loss = 1.128, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 02:19:25.219449 139795539912448 logging_writer.py:48] [35500] global_step=35500, grad_norm=0.482271, loss=1.09501 +I0917 02:19:25.227139 139813834572992 submission.py:307] 35500) loss = 1.095, grad_norm = 0.482 +I0917 02:27:08.842106 139795531519744 logging_writer.py:48] [36000] global_step=36000, grad_norm=0.499999, loss=1.12982 +I0917 02:27:08.846216 139813834572992 submission.py:307] 36000) loss = 1.130, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 02:31:34.741322 139795539912448 logging_writer.py:48] [36500] global_step=36500, grad_norm=0.491209, loss=1.12353 +I0917 02:31:34.749387 139813834572992 submission.py:307] 36500) loss = 1.124, grad_norm = 0.491 +I0917 02:33:34.368653 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 02:33:52.373078 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 02:34:12.391427 139813834572992 spec.py:363] Evaluating on the test split. +I0917 02:34:22.692681 139813834572992 submission_runner.py:516] Time since start: 28813.03s, Step: 36676, {'train/ctc_loss': 0.1571575696981423, 'train/wer': 0.05989320483183017, 'validation/ctc_loss': 0.39315141643748747, 'validation/wer': 0.11189108289480036, 'validation/num_examples': 5348, 'test/ctc_loss': 0.21713411953997033, 'test/wer': 0.06934373286210468, 'test/num_examples': 2472, 'score': 27943.172302246094, 'total_duration': 28813.034168481827, 'accumulated_submission_time': 27943.172302246094, 'accumulated_eval_time': 815.4270529747009, 'accumulated_logging_time': 3.0661959648132324} +I0917 02:34:22.961632 139795539912448 logging_writer.py:48] [36676] accumulated_eval_time=815.427, accumulated_logging_time=3.0662, accumulated_submission_time=27943.2, global_step=36676, preemption_count=0, score=27943.2, test/ctc_loss=0.217134, test/num_examples=2472, test/wer=0.0693437, total_duration=28813, train/ctc_loss=0.157158, train/wer=0.0598932, validation/ctc_loss=0.393151, validation/num_examples=5348, validation/wer=0.111891 +I0917 02:39:50.193003 139795531519744 logging_writer.py:48] [37000] global_step=37000, grad_norm=0.499999, loss=1.09019 +I0917 02:39:50.197268 139813834572992 submission.py:307] 37000) loss = 1.090, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 02:44:32.410296 139795539912448 logging_writer.py:48] [37500] global_step=37500, grad_norm=0.387921, loss=1.17781 +I0917 02:44:32.417870 139813834572992 submission.py:307] 37500) loss = 1.178, grad_norm = 0.388 +I0917 02:51:39.404063 139795531519744 logging_writer.py:48] [38000] global_step=38000, grad_norm=0.499999, loss=1.11994 +I0917 02:51:39.408293 139813834572992 submission.py:307] 38000) loss = 1.120, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 02:56:38.438758 139795539912448 logging_writer.py:48] [38500] global_step=38500, grad_norm=0.499999, loss=1.11586 +I0917 02:56:38.446361 139813834572992 submission.py:307] 38500) loss = 1.116, grad_norm = 0.500 +I0917 03:03:28.166002 139795531519744 logging_writer.py:48] [39000] global_step=39000, grad_norm=0.499999, loss=1.09385 +I0917 03:03:28.170222 139813834572992 submission.py:307] 39000) loss = 1.094, grad_norm = 0.500 +I0917 03:03:31.234598 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 03:03:47.220710 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 03:04:07.015558 139813834572992 spec.py:363] Evaluating on the test split. +I0917 03:04:17.280387 139813834572992 submission_runner.py:516] Time since start: 30607.62s, Step: 39002, {'train/ctc_loss': 0.1511255163255616, 'train/wer': 0.05782024983991578, 'validation/ctc_loss': 0.3916050698447232, 'validation/wer': 0.11021097861246561, 'validation/num_examples': 5348, 'test/ctc_loss': 0.21718513089554164, 'test/wer': 0.0677391180712124, 'test/num_examples': 2472, 'score': 29688.638763189316, 'total_duration': 30607.621863126755, 'accumulated_submission_time': 29688.638763189316, 'accumulated_eval_time': 861.4726457595825, 'accumulated_logging_time': 3.345543146133423} +I0917 03:04:17.375125 139795539912448 logging_writer.py:48] [39002] accumulated_eval_time=861.473, accumulated_logging_time=3.34554, accumulated_submission_time=29688.6, global_step=39002, preemption_count=0, score=29688.6, test/ctc_loss=0.217185, test/num_examples=2472, test/wer=0.0677391, total_duration=30607.6, train/ctc_loss=0.151126, train/wer=0.0578202, validation/ctc_loss=0.391605, validation/num_examples=5348, validation/wer=0.110211 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 03:09:32.670015 139795539912448 logging_writer.py:48] [39500] global_step=39500, grad_norm=0.488162, loss=1.06131 +I0917 03:09:32.677599 139813834572992 submission.py:307] 39500) loss = 1.061, grad_norm = 0.488 +I0917 03:16:14.391149 139795531519744 logging_writer.py:48] [40000] global_step=40000, grad_norm=0.432772, loss=1.17756 +I0917 03:16:14.395425 139813834572992 submission.py:307] 40000) loss = 1.178, grad_norm = 0.433 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 03:21:47.938915 139795539912448 logging_writer.py:48] [40500] global_step=40500, grad_norm=0.499999, loss=1.10319 +I0917 03:21:47.954334 139813834572992 submission.py:307] 40500) loss = 1.103, grad_norm = 0.500 +I0917 03:28:05.412942 139795531519744 logging_writer.py:48] [41000] global_step=41000, grad_norm=0.499999, loss=1.12543 +I0917 03:28:05.417151 139813834572992 submission.py:307] 41000) loss = 1.125, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 03:33:26.142287 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +I0917 03:33:45.339894 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 03:34:05.106785 139813834572992 spec.py:363] Evaluating on the test split. +I0917 03:34:15.398977 139813834572992 submission_runner.py:516] Time since start: 32405.74s, Step: 41409, {'train/ctc_loss': 0.14402947216721132, 'train/wer': 0.05490074778324054, 'validation/ctc_loss': 0.38445943135297367, 'validation/wer': 0.10806739728672814, 'validation/num_examples': 5348, 'test/ctc_loss': 0.21215761399929028, 'test/wer': 0.06702821278410823, 'test/num_examples': 2472, 'score': 31433.99880838394, 'total_duration': 32405.74048280716, 'accumulated_submission_time': 31433.99880838394, 'accumulated_eval_time': 910.729204416275, 'accumulated_logging_time': 3.4501547813415527} +I0917 03:34:15.666981 139795539912448 logging_writer.py:48] [41409] accumulated_eval_time=910.729, accumulated_logging_time=3.45015, accumulated_submission_time=31434, global_step=41409, preemption_count=0, score=31434, test/ctc_loss=0.212158, test/num_examples=2472, test/wer=0.0670282, total_duration=32405.7, train/ctc_loss=0.144029, train/wer=0.0549007, validation/ctc_loss=0.384459, validation/num_examples=5348, validation/wer=0.108067 +I0917 03:34:58.802329 139795531519744 logging_writer.py:48] [41500] global_step=41500, grad_norm=0.499999, loss=1.18196 +I0917 03:34:58.805707 139813834572992 submission.py:307] 41500) loss = 1.182, grad_norm = 0.500 +I0917 03:41:09.648576 139795539912448 logging_writer.py:48] [42000] global_step=42000, grad_norm=0.483064, loss=1.14985 +I0917 03:41:09.652728 139813834572992 submission.py:307] 42000) loss = 1.150, grad_norm = 0.483 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 03:47:21.970458 139795539912448 logging_writer.py:48] [42500] global_step=42500, grad_norm=0.499999, loss=1.12684 +I0917 03:47:21.977930 139813834572992 submission.py:307] 42500) loss = 1.127, grad_norm = 0.500 +I0917 03:52:58.368083 139795531519744 logging_writer.py:48] [43000] global_step=43000, grad_norm=0.479601, loss=1.08723 +I0917 03:52:58.372195 139813834572992 submission.py:307] 43000) loss = 1.087, grad_norm = 0.480 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 03:59:34.823781 139795539912448 logging_writer.py:48] [43500] global_step=43500, grad_norm=0.499999, loss=1.12921 +I0917 03:59:34.831490 139813834572992 submission.py:307] 43500) loss = 1.129, grad_norm = 0.500 +I0917 04:03:24.507616 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 04:03:40.752677 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 04:04:01.262507 139813834572992 spec.py:363] Evaluating on the test split. +I0917 04:04:12.644415 139813834572992 submission_runner.py:516] Time since start: 34202.99s, Step: 43899, {'train/ctc_loss': 0.13634578953349263, 'train/wer': 0.05233940025396412, 'validation/ctc_loss': 0.37550437088733174, 'validation/wer': 0.10649350649350649, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2063891848869282, 'test/wer': 0.06580951800621535, 'test/num_examples': 2472, 'score': 33179.23314833641, 'total_duration': 34202.98544931412, 'accumulated_submission_time': 33179.23314833641, 'accumulated_eval_time': 958.8653280735016, 'accumulated_logging_time': 3.728386402130127} +I0917 04:04:12.924438 139795539912448 logging_writer.py:48] [43899] accumulated_eval_time=958.865, accumulated_logging_time=3.72839, accumulated_submission_time=33179.2, global_step=43899, preemption_count=0, score=33179.2, test/ctc_loss=0.206389, test/num_examples=2472, test/wer=0.0658095, total_duration=34203, train/ctc_loss=0.136346, train/wer=0.0523394, validation/ctc_loss=0.375504, validation/num_examples=5348, validation/wer=0.106494 +I0917 04:05:44.521263 139795531519744 logging_writer.py:48] [44000] global_step=44000, grad_norm=0.495394, loss=1.07335 +I0917 04:05:44.525034 139813834572992 submission.py:307] 44000) loss = 1.073, grad_norm = 0.495 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 04:12:31.480389 139795539912448 logging_writer.py:48] [44500] global_step=44500, grad_norm=0.499999, loss=1.10271 +I0917 04:12:31.487901 139813834572992 submission.py:307] 44500) loss = 1.103, grad_norm = 0.500 +I0917 04:17:36.249341 139795531519744 logging_writer.py:48] [45000] global_step=45000, grad_norm=0.499999, loss=1.08507 +I0917 04:17:36.253463 139813834572992 submission.py:307] 45000) loss = 1.085, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 04:24:40.769270 139795539912448 logging_writer.py:48] [45500] global_step=45500, grad_norm=0.458513, loss=1.05132 +I0917 04:24:40.778653 139813834572992 submission.py:307] 45500) loss = 1.051, grad_norm = 0.459 +I0917 04:29:31.865772 139795531519744 logging_writer.py:48] [46000] global_step=46000, grad_norm=0.499999, loss=0.992897 +I0917 04:29:31.870154 139813834572992 submission.py:307] 46000) loss = 0.993, grad_norm = 0.500 +I0917 04:33:21.331021 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 04:33:36.177320 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 04:33:55.940811 139813834572992 spec.py:363] Evaluating on the test split. +I0917 04:34:06.227439 139813834572992 submission_runner.py:516] Time since start: 35996.57s, Step: 46223, {'train/ctc_loss': 0.12780559946569967, 'train/wer': 0.049626108379730624, 'validation/ctc_loss': 0.3676198108957454, 'validation/wer': 0.10313329792883696, 'validation/num_examples': 5348, 'test/ctc_loss': 0.19977806027808245, 'test/wer': 0.06221436841143136, 'test/num_examples': 2472, 'score': 34924.75255584717, 'total_duration': 35996.56893110275, 'accumulated_submission_time': 34924.75255584717, 'accumulated_eval_time': 1003.7615249156952, 'accumulated_logging_time': 4.018621206283569} +I0917 04:34:06.357069 139795539912448 logging_writer.py:48] [46223] accumulated_eval_time=1003.76, accumulated_logging_time=4.01862, accumulated_submission_time=34924.8, global_step=46223, preemption_count=0, score=34924.8, test/ctc_loss=0.199778, test/num_examples=2472, test/wer=0.0622144, total_duration=35996.6, train/ctc_loss=0.127806, train/wer=0.0496261, validation/ctc_loss=0.36762, validation/num_examples=5348, validation/wer=0.103133 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 04:37:33.805377 139795539912448 logging_writer.py:48] [46500] global_step=46500, grad_norm=0.5, loss=1.01799 +I0917 04:37:33.812527 139813834572992 submission.py:307] 46500) loss = 1.018, grad_norm = 0.500 +I0917 04:42:24.604105 139795531519744 logging_writer.py:48] [47000] global_step=47000, grad_norm=0.454556, loss=1.07044 +I0917 04:42:24.609099 139813834572992 submission.py:307] 47000) loss = 1.070, grad_norm = 0.455 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 04:49:53.070108 139795539912448 logging_writer.py:48] [47500] global_step=47500, grad_norm=0.499999, loss=1.10447 +I0917 04:49:53.077626 139813834572992 submission.py:307] 47500) loss = 1.104, grad_norm = 0.500 +I0917 04:54:19.866366 139795531519744 logging_writer.py:48] [48000] global_step=48000, grad_norm=0.499999, loss=1.0236 +I0917 04:54:19.870822 139813834572992 submission.py:307] 48000) loss = 1.024, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 05:02:03.816740 139795539912448 logging_writer.py:48] [48500] global_step=48500, grad_norm=0.499999, loss=1.07512 +I0917 05:02:03.824025 139813834572992 submission.py:307] 48500) loss = 1.075, grad_norm = 0.500 +I0917 05:03:15.069219 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 05:03:34.079845 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 05:03:53.750978 139813834572992 spec.py:363] Evaluating on the test split. +I0917 05:04:04.008307 139813834572992 submission_runner.py:516] Time since start: 37794.35s, Step: 48661, {'train/ctc_loss': 0.1207196938334105, 'train/wer': 0.04643527713563204, 'validation/ctc_loss': 0.36337352014579566, 'validation/wer': 0.10234152464635736, 'validation/num_examples': 5348, 'test/ctc_loss': 0.19679589963465385, 'test/wer': 0.062417484207746836, 'test/num_examples': 2472, 'score': 36669.94425535202, 'total_duration': 37794.34977936745, 'accumulated_submission_time': 36669.94425535202, 'accumulated_eval_time': 1052.7004284858704, 'accumulated_logging_time': 4.158437013626099} +I0917 05:04:04.310977 139795539912448 logging_writer.py:48] [48661] accumulated_eval_time=1052.7, accumulated_logging_time=4.15844, accumulated_submission_time=36669.9, global_step=48661, preemption_count=0, score=36669.9, test/ctc_loss=0.196796, test/num_examples=2472, test/wer=0.0624175, total_duration=37794.3, train/ctc_loss=0.12072, train/wer=0.0464353, validation/ctc_loss=0.363374, validation/num_examples=5348, validation/wer=0.102342 +I0917 05:07:25.220736 139795531519744 logging_writer.py:48] [49000] global_step=49000, grad_norm=0.499999, loss=1.00537 +I0917 05:07:25.225065 139813834572992 submission.py:307] 49000) loss = 1.005, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 05:15:18.321038 139795539912448 logging_writer.py:48] [49500] global_step=49500, grad_norm=0.499999, loss=1.11963 +I0917 05:15:18.328354 139813834572992 submission.py:307] 49500) loss = 1.120, grad_norm = 0.500 +I0917 05:19:27.477007 139795531519744 logging_writer.py:48] [50000] global_step=50000, grad_norm=0.454486, loss=1.03998 +I0917 05:19:27.481186 139813834572992 submission.py:307] 50000) loss = 1.040, grad_norm = 0.454 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 05:27:27.499348 139795539912448 logging_writer.py:48] [50500] global_step=50500, grad_norm=0.499999, loss=1.02156 +I0917 05:27:27.507556 139813834572992 submission.py:307] 50500) loss = 1.022, grad_norm = 0.500 +I0917 05:31:29.286783 139795531519744 logging_writer.py:48] [51000] global_step=51000, grad_norm=0.499999, loss=1.03266 +I0917 05:31:29.290891 139813834572992 submission.py:307] 51000) loss = 1.033, grad_norm = 0.500 +I0917 05:33:13.033691 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 05:33:29.048170 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 05:33:48.679474 139813834572992 spec.py:363] Evaluating on the test split. +I0917 05:33:58.612432 139813834572992 submission_runner.py:516] Time since start: 39588.95s, Step: 51140, {'train/ctc_loss': 0.11244541565422218, 'train/wer': 0.04343437632272979, 'validation/ctc_loss': 0.3598224439845163, 'validation/wer': 0.09998551634239367, 'validation/num_examples': 5348, 'test/ctc_loss': 0.19321575434302213, 'test/wer': 0.05998009465196108, 'test/num_examples': 2472, 'score': 38414.99412441254, 'total_duration': 39588.95394158363, 'accumulated_submission_time': 38414.99412441254, 'accumulated_eval_time': 1098.2790446281433, 'accumulated_logging_time': 4.471155881881714} +I0917 05:33:58.928144 139795539912448 logging_writer.py:48] [51140] accumulated_eval_time=1098.28, accumulated_logging_time=4.47116, accumulated_submission_time=38415, global_step=51140, preemption_count=0, score=38415, test/ctc_loss=0.193216, test/num_examples=2472, test/wer=0.0599801, total_duration=39589, train/ctc_loss=0.112445, train/wer=0.0434344, validation/ctc_loss=0.359822, validation/num_examples=5348, validation/wer=0.0999855 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 05:40:20.593678 139795539912448 logging_writer.py:48] [51500] global_step=51500, grad_norm=0.499999, loss=1.09382 +I0917 05:40:20.601338 139813834572992 submission.py:307] 51500) loss = 1.094, grad_norm = 0.500 +I0917 05:44:19.385508 139795531519744 logging_writer.py:48] [52000] global_step=52000, grad_norm=0.485572, loss=1.01755 +I0917 05:44:19.390439 139813834572992 submission.py:307] 52000) loss = 1.018, grad_norm = 0.486 +I0917 05:52:11.195265 139795539912448 logging_writer.py:48] [52500] global_step=52500, grad_norm=0.499999, loss=0.977819 +I0917 05:52:11.199488 139813834572992 submission.py:307] 52500) loss = 0.978, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 05:56:21.779903 139795539912448 logging_writer.py:48] [53000] global_step=53000, grad_norm=0.499999, loss=1.06258 +I0917 05:56:21.797671 139813834572992 submission.py:307] 53000) loss = 1.063, grad_norm = 0.500 +I0917 06:03:08.063720 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 06:03:22.478076 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 06:03:42.964486 139813834572992 spec.py:363] Evaluating on the test split. +I0917 06:03:54.266564 139813834572992 submission_runner.py:516] Time since start: 41384.61s, Step: 53453, {'train/ctc_loss': 0.10562311169952113, 'train/wer': 0.04101411997091351, 'validation/ctc_loss': 0.3530767163718731, 'validation/wer': 0.09794814850576933, 'validation/num_examples': 5348, 'test/ctc_loss': 0.18932752465884894, 'test/wer': 0.05904576198890988, 'test/num_examples': 2472, 'score': 40161.328478097916, 'total_duration': 41384.607890844345, 'accumulated_submission_time': 40161.328478097916, 'accumulated_eval_time': 1144.4815368652344, 'accumulated_logging_time': 4.797126770019531} +I0917 06:03:54.348268 139795539912448 logging_writer.py:48] [53453] accumulated_eval_time=1144.48, accumulated_logging_time=4.79713, accumulated_submission_time=40161.3, global_step=53453, preemption_count=0, score=40161.3, test/ctc_loss=0.189328, test/num_examples=2472, test/wer=0.0590458, total_duration=41384.6, train/ctc_loss=0.105623, train/wer=0.0410141, validation/ctc_loss=0.353077, validation/num_examples=5348, validation/wer=0.0979481 +I0917 06:04:44.927276 139795531519744 logging_writer.py:48] [53500] global_step=53500, grad_norm=0.499999, loss=1.03169 +I0917 06:04:44.930652 139813834572992 submission.py:307] 53500) loss = 1.032, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 06:09:21.794596 139795539912448 logging_writer.py:48] [54000] global_step=54000, grad_norm=0.499999, loss=1.06434 +I0917 06:09:21.802436 139813834572992 submission.py:307] 54000) loss = 1.064, grad_norm = 0.500 +I0917 06:16:46.758706 139795531519744 logging_writer.py:48] [54500] global_step=54500, grad_norm=0.499999, loss=1.0277 +I0917 06:16:46.762925 139813834572992 submission.py:307] 54500) loss = 1.028, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 06:21:33.261351 139795539912448 logging_writer.py:48] [55000] global_step=55000, grad_norm=0.499999, loss=0.991956 +I0917 06:21:33.268586 139813834572992 submission.py:307] 55000) loss = 0.992, grad_norm = 0.500 +I0917 06:28:38.348907 139795531519744 logging_writer.py:48] [55500] global_step=55500, grad_norm=0.387949, loss=0.981867 +I0917 06:28:38.353219 139813834572992 submission.py:307] 55500) loss = 0.982, grad_norm = 0.388 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 06:33:03.551400 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +I0917 06:33:18.874617 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 06:33:38.605269 139813834572992 spec.py:363] Evaluating on the test split. +I0917 06:33:49.053582 139813834572992 submission_runner.py:516] Time since start: 43179.40s, Step: 55898, {'train/ctc_loss': 0.09911215818381823, 'train/wer': 0.037920967234287326, 'validation/ctc_loss': 0.3445919098948915, 'validation/wer': 0.09508038429971516, 'validation/num_examples': 5348, 'test/ctc_loss': 0.1878857229700303, 'test/wer': 0.05723803140170211, 'test/num_examples': 2472, 'score': 41906.71695399284, 'total_duration': 43179.3950483799, 'accumulated_submission_time': 41906.71695399284, 'accumulated_eval_time': 1189.9835596084595, 'accumulated_logging_time': 4.889002799987793} +I0917 06:33:49.320732 139795539912448 logging_writer.py:48] [55898] accumulated_eval_time=1189.98, accumulated_logging_time=4.889, accumulated_submission_time=41906.7, global_step=55898, preemption_count=0, score=41906.7, test/ctc_loss=0.187886, test/num_examples=2472, test/wer=0.057238, total_duration=43179.4, train/ctc_loss=0.0991122, train/wer=0.037921, validation/ctc_loss=0.344592, validation/num_examples=5348, validation/wer=0.0950804 +I0917 06:34:42.594481 139795531519744 logging_writer.py:48] [56000] global_step=56000, grad_norm=0.499999, loss=0.974587 +I0917 06:34:42.597981 139813834572992 submission.py:307] 56000) loss = 0.975, grad_norm = 0.500 +I0917 06:41:35.434619 139795539912448 logging_writer.py:48] [56500] global_step=56500, grad_norm=0.499999, loss=1.03787 +I0917 06:41:35.438750 139813834572992 submission.py:307] 56500) loss = 1.038, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 06:46:56.907774 139795539912448 logging_writer.py:48] [57000] global_step=57000, grad_norm=0.499999, loss=1.01872 +I0917 06:46:56.915431 139813834572992 submission.py:307] 57000) loss = 1.019, grad_norm = 0.500 +I0917 06:53:23.343993 139795531519744 logging_writer.py:48] [57500] global_step=57500, grad_norm=0.499999, loss=0.981908 +I0917 06:53:23.348145 139813834572992 submission.py:307] 57500) loss = 0.982, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 06:59:06.498661 139795539912448 logging_writer.py:48] [58000] global_step=58000, grad_norm=0.499999, loss=1.03608 +I0917 06:59:06.506151 139813834572992 submission.py:307] 58000) loss = 1.036, grad_norm = 0.500 +I0917 07:02:57.942340 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 07:03:13.878495 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 07:03:33.742999 139813834572992 spec.py:363] Evaluating on the test split. +I0917 07:03:44.240793 139813834572992 submission_runner.py:516] Time since start: 44974.58s, Step: 58362, {'train/ctc_loss': 0.09305893007712039, 'train/wer': 0.036542614962176714, 'validation/ctc_loss': 0.3463119558955571, 'validation/wer': 0.09385410128904553, 'validation/num_examples': 5348, 'test/ctc_loss': 0.18418335457126267, 'test/wer': 0.05603964820344078, 'test/num_examples': 2472, 'score': 43651.70753669739, 'total_duration': 44974.58230113983, 'accumulated_submission_time': 43651.70753669739, 'accumulated_eval_time': 1236.2817873954773, 'accumulated_logging_time': 5.1664628982543945} +I0917 07:03:44.537292 139795539912448 logging_writer.py:48] [58362] accumulated_eval_time=1236.28, accumulated_logging_time=5.16646, accumulated_submission_time=43651.7, global_step=58362, preemption_count=0, score=43651.7, test/ctc_loss=0.184183, test/num_examples=2472, test/wer=0.0560396, total_duration=44974.6, train/ctc_loss=0.0930589, train/wer=0.0365426, validation/ctc_loss=0.346312, validation/num_examples=5348, validation/wer=0.0938541 +I0917 07:06:02.093300 139795531519744 logging_writer.py:48] [58500] global_step=58500, grad_norm=0.499999, loss=1.03228 +I0917 07:06:02.097602 139813834572992 submission.py:307] 58500) loss = 1.032, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 07:11:58.699974 139795539912448 logging_writer.py:48] [59000] global_step=59000, grad_norm=0.499999, loss=0.990147 +I0917 07:11:58.709241 139813834572992 submission.py:307] 59000) loss = 0.990, grad_norm = 0.500 +I0917 07:17:48.172359 139795531519744 logging_writer.py:48] [59500] global_step=59500, grad_norm=0.499999, loss=1.01481 +I0917 07:17:48.176828 139813834572992 submission.py:307] 59500) loss = 1.015, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 07:24:10.627014 139795539912448 logging_writer.py:48] [60000] global_step=60000, grad_norm=0.499999, loss=1.02254 +I0917 07:24:10.643841 139813834572992 submission.py:307] 60000) loss = 1.023, grad_norm = 0.500 +I0917 07:29:40.431354 139795531519744 logging_writer.py:48] [60500] global_step=60500, grad_norm=0.499999, loss=0.961099 +I0917 07:29:40.435744 139813834572992 submission.py:307] 60500) loss = 0.961, grad_norm = 0.500 +I0917 07:32:54.572705 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 07:33:09.062631 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 07:33:28.887194 139813834572992 spec.py:363] Evaluating on the test split. +I0917 07:33:39.075117 139813834572992 submission_runner.py:516] Time since start: 46769.42s, Step: 60677, {'train/ctc_loss': 0.08922444183189648, 'train/wer': 0.034686723320201, 'validation/ctc_loss': 0.34472933403813794, 'validation/wer': 0.09249263747405011, 'validation/num_examples': 5348, 'test/ctc_loss': 0.18443950769001227, 'test/wer': 0.05569435134970447, 'test/num_examples': 2472, 'score': 45398.96501159668, 'total_duration': 46769.416610479355, 'accumulated_submission_time': 45398.96501159668, 'accumulated_eval_time': 1280.783971786499, 'accumulated_logging_time': 5.473428726196289} +I0917 07:33:39.178753 139795539912448 logging_writer.py:48] [60677] accumulated_eval_time=1280.78, accumulated_logging_time=5.47343, accumulated_submission_time=45399, global_step=60677, preemption_count=0, score=45399, test/ctc_loss=0.18444, test/num_examples=2472, test/wer=0.0556944, total_duration=46769.4, train/ctc_loss=0.0892244, train/wer=0.0346867, validation/ctc_loss=0.344729, validation/num_examples=5348, validation/wer=0.0924926 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 07:37:02.483912 139795539912448 logging_writer.py:48] [61000] global_step=61000, grad_norm=0.499999, loss=1.03094 +I0917 07:37:02.494678 139813834572992 submission.py:307] 61000) loss = 1.031, grad_norm = 0.500 +I0917 07:42:28.801893 139795531519744 logging_writer.py:48] [61500] global_step=61500, grad_norm=0.499999, loss=0.974639 +I0917 07:42:28.806630 139813834572992 submission.py:307] 61500) loss = 0.975, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 07:49:21.166724 139795539912448 logging_writer.py:48] [62000] global_step=62000, grad_norm=0.499999, loss=1.00133 +I0917 07:49:21.174175 139813834572992 submission.py:307] 62000) loss = 1.001, grad_norm = 0.500 +I0917 07:54:22.722332 139795531519744 logging_writer.py:48] [62500] global_step=62500, grad_norm=0.483728, loss=1.00575 +I0917 07:54:22.726890 139813834572992 submission.py:307] 62500) loss = 1.006, grad_norm = 0.484 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 08:01:35.701970 139795539912448 logging_writer.py:48] [63000] global_step=63000, grad_norm=0.499999, loss=0.960331 +I0917 08:01:35.709630 139813834572992 submission.py:307] 63000) loss = 0.960, grad_norm = 0.500 +I0917 08:02:47.944789 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 08:03:03.317123 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 08:03:23.761398 139813834572992 spec.py:363] Evaluating on the test split. +I0917 08:03:34.096149 139813834572992 submission_runner.py:516] Time since start: 48564.44s, Step: 63162, {'train/ctc_loss': 0.08445938078740862, 'train/wer': 0.03309673428190017, 'validation/ctc_loss': 0.3397519458650417, 'validation/wer': 0.09125669869164293, 'validation/num_examples': 5348, 'test/ctc_loss': 0.18078597056664947, 'test/wer': 0.05376475128470741, 'test/num_examples': 2472, 'score': 47144.11619091034, 'total_duration': 48564.43762612343, 'accumulated_submission_time': 47144.11619091034, 'accumulated_eval_time': 1326.9350910186768, 'accumulated_logging_time': 5.58700704574585} +I0917 08:03:34.388400 139795539912448 logging_writer.py:48] [63162] accumulated_eval_time=1326.94, accumulated_logging_time=5.58701, accumulated_submission_time=47144.1, global_step=63162, preemption_count=0, score=47144.1, test/ctc_loss=0.180786, test/num_examples=2472, test/wer=0.0537648, total_duration=48564.4, train/ctc_loss=0.0844594, train/wer=0.0330967, validation/ctc_loss=0.339752, validation/num_examples=5348, validation/wer=0.0912567 +I0917 08:07:22.001939 139795531519744 logging_writer.py:48] [63500] global_step=63500, grad_norm=0.477188, loss=0.998919 +I0917 08:07:22.006260 139813834572992 submission.py:307] 63500) loss = 0.999, grad_norm = 0.477 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 08:14:47.043830 139795539912448 logging_writer.py:48] [64000] global_step=64000, grad_norm=0.499999, loss=1.00421 +I0917 08:14:47.051395 139813834572992 submission.py:307] 64000) loss = 1.004, grad_norm = 0.500 +I0917 08:19:19.418174 139795531519744 logging_writer.py:48] [64500] global_step=64500, grad_norm=0.499999, loss=0.951125 +I0917 08:19:19.422460 139813834572992 submission.py:307] 64500) loss = 0.951, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 08:26:55.231515 139795539912448 logging_writer.py:48] [65000] global_step=65000, grad_norm=0.499999, loss=0.964394 +I0917 08:26:55.238922 139813834572992 submission.py:307] 65000) loss = 0.964, grad_norm = 0.500 +I0917 08:31:20.792003 139795531519744 logging_writer.py:48] [65500] global_step=65500, grad_norm=0.499999, loss=0.913634 +I0917 08:31:20.796192 139813834572992 submission.py:307] 65500) loss = 0.914, grad_norm = 0.500 +I0917 08:32:43.962449 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 08:33:00.647488 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 08:33:21.161652 139813834572992 spec.py:363] Evaluating on the test split. +I0917 08:33:31.526833 139813834572992 submission_runner.py:516] Time since start: 50361.87s, Step: 65602, {'train/ctc_loss': 0.08036556805527863, 'train/wer': 0.031533878162341676, 'validation/ctc_loss': 0.3392857633394866, 'validation/wer': 0.08989523487664752, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17962803295777147, 'test/wer': 0.05335851969207645, 'test/num_examples': 2472, 'score': 48890.078708171844, 'total_duration': 50361.868340969086, 'accumulated_submission_time': 48890.078708171844, 'accumulated_eval_time': 1374.4993312358856, 'accumulated_logging_time': 5.889404773712158} +I0917 08:33:31.825202 139795539912448 logging_writer.py:48] [65602] accumulated_eval_time=1374.5, accumulated_logging_time=5.8894, accumulated_submission_time=48890.1, global_step=65602, preemption_count=0, score=48890.1, test/ctc_loss=0.179628, test/num_examples=2472, test/wer=0.0533585, total_duration=50361.9, train/ctc_loss=0.0803656, train/wer=0.0315339, validation/ctc_loss=0.339286, validation/num_examples=5348, validation/wer=0.0898952 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 08:39:47.929336 139795539912448 logging_writer.py:48] [66000] global_step=66000, grad_norm=0.499999, loss=0.927556 +I0917 08:39:47.948073 139813834572992 submission.py:307] 66000) loss = 0.928, grad_norm = 0.500 +I0917 08:44:07.741302 139795531519744 logging_writer.py:48] [66500] global_step=66500, grad_norm=0.499999, loss=0.959845 +I0917 08:44:07.745643 139813834572992 submission.py:307] 66500) loss = 0.960, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 08:51:58.265022 139795539912448 logging_writer.py:48] [67000] global_step=67000, grad_norm=0.499999, loss=0.916622 +I0917 08:51:58.272326 139813834572992 submission.py:307] 67000) loss = 0.917, grad_norm = 0.500 +I0917 08:56:02.178379 139795531519744 logging_writer.py:48] [67500] global_step=67500, grad_norm=0.499999, loss=0.920132 +I0917 08:56:02.182588 139813834572992 submission.py:307] 67500) loss = 0.920, grad_norm = 0.500 +I0917 09:02:41.733941 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 09:02:56.618726 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 09:03:17.164057 139813834572992 spec.py:363] Evaluating on the test split. +I0917 09:03:28.432035 139813834572992 submission_runner.py:516] Time since start: 52158.77s, Step: 67911, {'train/ctc_loss': 0.07766145285923333, 'train/wer': 0.030264057565200404, 'validation/ctc_loss': 0.3344761390678371, 'validation/wer': 0.08920967508328104, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17733108537244338, 'test/wer': 0.05238356386976215, 'test/num_examples': 2472, 'score': 50637.222826480865, 'total_duration': 52158.77340841293, 'accumulated_submission_time': 50637.222826480865, 'accumulated_eval_time': 1421.1971504688263, 'accumulated_logging_time': 6.198186874389648} +I0917 09:03:28.506223 139795539912448 logging_writer.py:48] [67911] accumulated_eval_time=1421.2, accumulated_logging_time=6.19819, accumulated_submission_time=50637.2, global_step=67911, preemption_count=0, score=50637.2, test/ctc_loss=0.177331, test/num_examples=2472, test/wer=0.0523836, total_duration=52158.8, train/ctc_loss=0.0776615, train/wer=0.0302641, validation/ctc_loss=0.334476, validation/num_examples=5348, validation/wer=0.0892097 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 09:04:55.495315 139795539912448 logging_writer.py:48] [68000] global_step=68000, grad_norm=0.5, loss=0.911689 +I0917 09:04:55.502421 139813834572992 submission.py:307] 68000) loss = 0.912, grad_norm = 0.500 +I0917 09:09:03.142754 139795531519744 logging_writer.py:48] [68500] global_step=68500, grad_norm=0.499999, loss=0.854012 +I0917 09:09:03.147054 139813834572992 submission.py:307] 68500) loss = 0.854, grad_norm = 0.500 +I0917 09:17:08.528484 139795539912448 logging_writer.py:48] [69000] global_step=69000, grad_norm=0.499999, loss=0.995109 +I0917 09:17:08.532694 139813834572992 submission.py:307] 69000) loss = 0.995, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 09:21:05.612301 139795539912448 logging_writer.py:48] [69500] global_step=69500, grad_norm=0.499999, loss=0.942529 +I0917 09:21:05.619527 139813834572992 submission.py:307] 69500) loss = 0.943, grad_norm = 0.500 +I0917 09:28:59.719401 139795531519744 logging_writer.py:48] [70000] global_step=70000, grad_norm=0.5, loss=1.05659 +I0917 09:28:59.723820 139813834572992 submission.py:307] 70000) loss = 1.057, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 09:32:37.608810 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +I0917 09:32:53.161962 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 09:33:13.530006 139813834572992 spec.py:363] Evaluating on the test split. +I0917 09:33:23.971598 139813834572992 submission_runner.py:516] Time since start: 53954.31s, Step: 70423, {'train/ctc_loss': 0.07588853751972788, 'train/wer': 0.029466349754175756, 'validation/ctc_loss': 0.335301566895218, 'validation/wer': 0.08888137884420412, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17690643599990322, 'test/wer': 0.05226169439197286, 'test/num_examples': 2472, 'score': 52382.57315540314, 'total_duration': 53954.31312060356, 'accumulated_submission_time': 52382.57315540314, 'accumulated_eval_time': 1467.5598187446594, 'accumulated_logging_time': 6.286574602127075} +I0917 09:33:24.330493 139795539912448 logging_writer.py:48] [70423] accumulated_eval_time=1467.56, accumulated_logging_time=6.28657, accumulated_submission_time=52382.6, global_step=70423, preemption_count=0, score=52382.6, test/ctc_loss=0.176906, test/num_examples=2472, test/wer=0.0522617, total_duration=53954.3, train/ctc_loss=0.0758885, train/wer=0.0294663, validation/ctc_loss=0.335302, validation/num_examples=5348, validation/wer=0.0888814 +I0917 09:34:09.925821 139795531519744 logging_writer.py:48] [70500] global_step=70500, grad_norm=0.499999, loss=0.955784 +I0917 09:34:09.929242 139813834572992 submission.py:307] 70500) loss = 0.956, grad_norm = 0.500 +I0917 09:41:45.727686 139795539912448 logging_writer.py:48] [71000] global_step=71000, grad_norm=0.5, loss=0.924828 +I0917 09:41:45.731911 139813834572992 submission.py:307] 71000) loss = 0.925, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 09:46:17.619224 139795539912448 logging_writer.py:48] [71500] global_step=71500, grad_norm=0.499999, loss=0.963127 +I0917 09:46:17.626477 139813834572992 submission.py:307] 71500) loss = 0.963, grad_norm = 0.500 +I0917 09:53:32.878676 139795531519744 logging_writer.py:48] [72000] global_step=72000, grad_norm=0.499999, loss=0.941566 +I0917 09:53:32.883220 139813834572992 submission.py:307] 72000) loss = 0.942, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 09:58:24.899774 139795539912448 logging_writer.py:48] [72500] global_step=72500, grad_norm=0.499999, loss=0.910371 +I0917 09:58:24.907534 139813834572992 submission.py:307] 72500) loss = 0.910, grad_norm = 0.500 +I0917 10:02:33.677268 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 10:02:50.424505 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 10:03:09.950858 139813834572992 spec.py:363] Evaluating on the test split. +I0917 10:03:20.133117 139813834572992 submission_runner.py:516] Time since start: 55750.47s, Step: 72847, {'train/ctc_loss': 0.07473460691878364, 'train/wer': 0.029086488891783068, 'validation/ctc_loss': 0.3327777993614376, 'validation/wer': 0.08807029401824941, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17564540454222852, 'test/wer': 0.05195702069749965, 'test/num_examples': 2472, 'score': 54128.103445768356, 'total_duration': 55750.47462272644, 'accumulated_submission_time': 54128.103445768356, 'accumulated_eval_time': 1514.0154871940613, 'accumulated_logging_time': 6.655657052993774} +I0917 10:03:20.396938 139795539912448 logging_writer.py:48] [72847] accumulated_eval_time=1514.02, accumulated_logging_time=6.65566, accumulated_submission_time=54128.1, global_step=72847, preemption_count=0, score=54128.1, test/ctc_loss=0.175645, test/num_examples=2472, test/wer=0.051957, total_duration=55750.5, train/ctc_loss=0.0747346, train/wer=0.0290865, validation/ctc_loss=0.332778, validation/num_examples=5348, validation/wer=0.0880703 +I0917 10:06:01.406531 139795531519744 logging_writer.py:48] [73000] global_step=73000, grad_norm=0.499999, loss=0.877207 +I0917 10:06:01.410804 139813834572992 submission.py:307] 73000) loss = 0.877, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 10:11:16.190953 139795539912448 logging_writer.py:48] [73500] global_step=73500, grad_norm=0.499999, loss=1.05306 +I0917 10:11:16.198319 139813834572992 submission.py:307] 73500) loss = 1.053, grad_norm = 0.500 +I0917 10:17:57.963665 139795531519744 logging_writer.py:48] [74000] global_step=74000, grad_norm=0.459383, loss=0.897503 +I0917 10:17:57.967882 139813834572992 submission.py:307] 74000) loss = 0.898, grad_norm = 0.459 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 10:23:24.873845 139795539912448 logging_writer.py:48] [74500] global_step=74500, grad_norm=0.499999, loss=0.945396 +I0917 10:23:24.882237 139813834572992 submission.py:307] 74500) loss = 0.945, grad_norm = 0.500 +I0917 10:29:46.812526 139795531519744 logging_writer.py:48] [75000] global_step=75000, grad_norm=0.499999, loss=0.96704 +I0917 10:29:46.816864 139813834572992 submission.py:307] 75000) loss = 0.967, grad_norm = 0.500 +I0917 10:32:31.037191 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 10:32:48.327848 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 10:33:08.057537 139813834572992 spec.py:363] Evaluating on the test split. +I0917 10:33:18.217476 139813834572992 submission_runner.py:516] Time since start: 57548.56s, Step: 75145, {'train/ctc_loss': 0.0745783904498464, 'train/wer': 0.02895082429807139, 'validation/ctc_loss': 0.3340026475302014, 'validation/wer': 0.08831168831168831, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17615674045502935, 'test/wer': 0.05222107123270977, 'test/num_examples': 2472, 'score': 55875.95197558403, 'total_duration': 57548.5589632988, 'accumulated_submission_time': 55875.95197558403, 'accumulated_eval_time': 1561.195592880249, 'accumulated_logging_time': 6.92978048324585} +I0917 10:33:18.284954 139795539912448 logging_writer.py:48] [75145] accumulated_eval_time=1561.2, accumulated_logging_time=6.92978, accumulated_submission_time=55876, global_step=75145, preemption_count=0, score=55876, test/ctc_loss=0.176157, test/num_examples=2472, test/wer=0.0522211, total_duration=57548.6, train/ctc_loss=0.0745784, train/wer=0.0289508, validation/ctc_loss=0.334003, validation/num_examples=5348, validation/wer=0.0883117 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 10:36:24.499698 139795539912448 logging_writer.py:48] [75500] global_step=75500, grad_norm=0.499999, loss=0.935525 +I0917 10:36:24.515847 139813834572992 submission.py:307] 75500) loss = 0.936, grad_norm = 0.500 +I0917 10:42:34.958204 139795531519744 logging_writer.py:48] [76000] global_step=76000, grad_norm=0.499999, loss=0.935645 +I0917 10:42:34.962445 139813834572992 submission.py:307] 76000) loss = 0.936, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 10:48:39.316502 139795539912448 logging_writer.py:48] [76500] global_step=76500, grad_norm=0.499999, loss=0.964588 +I0917 10:48:39.327144 139813834572992 submission.py:307] 76500) loss = 0.965, grad_norm = 0.500 +I0917 10:54:22.799276 139795531519744 logging_writer.py:48] [77000] global_step=77000, grad_norm=0.5, loss=0.956256 +I0917 10:54:22.803459 139813834572992 submission.py:307] 77000) loss = 0.956, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 11:00:53.540904 139795539912448 logging_writer.py:48] [77500] global_step=77500, grad_norm=0.5, loss=0.929888 +I0917 11:00:53.548746 139813834572992 submission.py:307] 77500) loss = 0.930, grad_norm = 0.500 +I0917 11:02:27.089044 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 11:02:42.711052 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 11:03:03.319783 139813834572992 spec.py:363] Evaluating on the test split. +I0917 11:03:14.578481 139813834572992 submission_runner.py:516] Time since start: 59344.92s, Step: 77691, {'train/ctc_loss': 0.07448023948465872, 'train/wer': 0.028874852125592855, 'validation/ctc_loss': 0.3337463660965441, 'validation/wer': 0.08841790180080143, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17598736208787663, 'test/wer': 0.05199764385676274, 'test/num_examples': 2472, 'score': 57621.09947395325, 'total_duration': 59344.91904091835, 'accumulated_submission_time': 57621.09947395325, 'accumulated_eval_time': 1608.6839032173157, 'accumulated_logging_time': 7.007215261459351} +I0917 11:03:14.908420 139795539912448 logging_writer.py:48] [77691] accumulated_eval_time=1608.68, accumulated_logging_time=7.00722, accumulated_submission_time=57621.1, global_step=77691, preemption_count=0, score=57621.1, test/ctc_loss=0.175987, test/num_examples=2472, test/wer=0.0519976, total_duration=59344.9, train/ctc_loss=0.0744802, train/wer=0.0288749, validation/ctc_loss=0.333746, validation/num_examples=5348, validation/wer=0.0884179 +I0917 11:07:16.649904 139795531519744 logging_writer.py:48] [78000] global_step=78000, grad_norm=0.499999, loss=0.907947 +I0917 11:07:16.654049 139813834572992 submission.py:307] 78000) loss = 0.908, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 11:14:01.488129 139795539912448 logging_writer.py:48] [78500] global_step=78500, grad_norm=0.499999, loss=0.962238 +I0917 11:14:01.495517 139813834572992 submission.py:307] 78500) loss = 0.962, grad_norm = 0.500 +I0917 11:19:06.635383 139795531519744 logging_writer.py:48] [79000] global_step=79000, grad_norm=0.499999, loss=0.975488 +I0917 11:19:06.639368 139813834572992 submission.py:307] 79000) loss = 0.975, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 11:26:09.322550 139795539912448 logging_writer.py:48] [79500] global_step=79500, grad_norm=0.499999, loss=0.864797 +I0917 11:26:09.334379 139813834572992 submission.py:307] 79500) loss = 0.865, grad_norm = 0.500 +I0917 11:31:02.580370 139795531519744 logging_writer.py:48] [80000] global_step=80000, grad_norm=0.5, loss=0.938919 +I0917 11:31:02.584589 139813834572992 submission.py:307] 80000) loss = 0.939, grad_norm = 0.500 +I0917 11:32:25.964309 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 11:32:43.175068 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 11:33:02.873081 139813834572992 spec.py:363] Evaluating on the test split. +I0917 11:33:13.023705 139813834572992 submission_runner.py:516] Time since start: 61143.37s, Step: 80087, {'train/ctc_loss': 0.07409676425772471, 'train/wer': 0.02873918753188118, 'validation/ctc_loss': 0.33345984301223125, 'validation/wer': 0.08790614589871096, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17555488710965222, 'test/wer': 0.05124611541039547, 'test/num_examples': 2472, 'score': 59368.39318227768, 'total_duration': 61143.365178346634, 'accumulated_submission_time': 59368.39318227768, 'accumulated_eval_time': 1655.7432885169983, 'accumulated_logging_time': 7.346552133560181} +I0917 11:33:13.438456 139795539912448 logging_writer.py:48] [80087] accumulated_eval_time=1655.74, accumulated_logging_time=7.34655, accumulated_submission_time=59368.4, global_step=80087, preemption_count=0, score=59368.4, test/ctc_loss=0.175555, test/num_examples=2472, test/wer=0.0512461, total_duration=61143.4, train/ctc_loss=0.0740968, train/wer=0.0287392, validation/ctc_loss=0.33346, validation/num_examples=5348, validation/wer=0.0879061 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 11:38:59.448411 139795539912448 logging_writer.py:48] [80500] global_step=80500, grad_norm=0.499999, loss=0.940966 +I0917 11:38:59.456340 139813834572992 submission.py:307] 80500) loss = 0.941, grad_norm = 0.500 +I0917 11:43:52.614778 139795531519744 logging_writer.py:48] [81000] global_step=81000, grad_norm=0.5, loss=0.920775 +I0917 11:43:52.619022 139813834572992 submission.py:307] 81000) loss = 0.921, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 11:51:15.834310 139795539912448 logging_writer.py:48] [81500] global_step=81500, grad_norm=0.499999, loss=0.939752 +I0917 11:51:15.842174 139813834572992 submission.py:307] 81500) loss = 0.940, grad_norm = 0.500 +I0917 11:55:48.508168 139795531519744 logging_writer.py:48] [82000] global_step=82000, grad_norm=0.499999, loss=0.998036 +I0917 11:55:48.512228 139813834572992 submission.py:307] 82000) loss = 0.998, grad_norm = 0.500 +I0917 12:02:21.384043 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 12:02:36.041865 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 12:02:55.630945 139813834572992 spec.py:363] Evaluating on the test split. +I0917 12:03:06.110717 139813834572992 submission_runner.py:516] Time since start: 62936.45s, Step: 82379, {'train/ctc_loss': 0.07365463607338266, 'train/wer': 0.028733760948132712, 'validation/ctc_loss': 0.33402197471650846, 'validation/wer': 0.08779993240959784, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17557619644815795, 'test/wer': 0.051713281741921066, 'test/num_examples': 2472, 'score': 61113.596465587616, 'total_duration': 62936.452160835266, 'accumulated_submission_time': 61113.596465587616, 'accumulated_eval_time': 1700.4697089195251, 'accumulated_logging_time': 7.771591901779175} +I0917 12:03:06.173308 139795539912448 logging_writer.py:48] [82379] accumulated_eval_time=1700.47, accumulated_logging_time=7.77159, accumulated_submission_time=61113.6, global_step=82379, preemption_count=0, score=61113.6, test/ctc_loss=0.175576, test/num_examples=2472, test/wer=0.0517133, total_duration=62936.5, train/ctc_loss=0.0736546, train/wer=0.0287338, validation/ctc_loss=0.334022, validation/num_examples=5348, validation/wer=0.0877999 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 12:04:11.445035 139795539912448 logging_writer.py:48] [82500] global_step=82500, grad_norm=0.499999, loss=0.985181 +I0917 12:04:11.452640 139813834572992 submission.py:307] 82500) loss = 0.985, grad_norm = 0.500 +I0917 12:08:43.532816 139795531519744 logging_writer.py:48] [83000] global_step=83000, grad_norm=0.499999, loss=0.912049 +I0917 12:08:43.537141 139813834572992 submission.py:307] 83000) loss = 0.912, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 12:16:31.725169 139795539912448 logging_writer.py:48] [83500] global_step=83500, grad_norm=0.499999, loss=0.976539 +I0917 12:16:31.732710 139813834572992 submission.py:307] 83500) loss = 0.977, grad_norm = 0.500 +I0917 12:20:43.788143 139795531519744 logging_writer.py:48] [84000] global_step=84000, grad_norm=0.5, loss=0.946983 +I0917 12:20:43.792587 139813834572992 submission.py:307] 84000) loss = 0.947, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 12:28:49.702776 139795539912448 logging_writer.py:48] [84500] global_step=84500, grad_norm=0.499999, loss=0.951509 +I0917 12:28:49.710278 139813834572992 submission.py:307] 84500) loss = 0.952, grad_norm = 0.500 +I0917 12:32:15.563408 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 12:32:32.533607 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 12:32:52.279610 139813834572992 spec.py:363] Evaluating on the test split. +I0917 12:33:02.691322 139813834572992 submission_runner.py:516] Time since start: 64733.03s, Step: 84938, {'train/ctc_loss': 0.07344040690551908, 'train/wer': 0.028733760948132712, 'validation/ctc_loss': 0.3364899435842375, 'validation/wer': 0.08833099985516342, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17640339195270663, 'test/wer': 0.051936709117868095, 'test/num_examples': 2472, 'score': 62859.03667855263, 'total_duration': 64733.032787799835, 'accumulated_submission_time': 62859.03667855263, 'accumulated_eval_time': 1747.5974349975586, 'accumulated_logging_time': 7.8443779945373535} +I0917 12:33:02.998925 139795539912448 logging_writer.py:48] [84938] accumulated_eval_time=1747.6, accumulated_logging_time=7.84438, accumulated_submission_time=62859, global_step=84938, preemption_count=0, score=62859, test/ctc_loss=0.176403, test/num_examples=2472, test/wer=0.0519367, total_duration=64733, train/ctc_loss=0.0734404, train/wer=0.0287338, validation/ctc_loss=0.33649, validation/num_examples=5348, validation/wer=0.088331 +I0917 12:33:46.570773 139795531519744 logging_writer.py:48] [85000] global_step=85000, grad_norm=0.499999, loss=0.960752 +I0917 12:33:46.574166 139813834572992 submission.py:307] 85000) loss = 0.961, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 12:41:54.642585 139795539912448 logging_writer.py:48] [85500] global_step=85500, grad_norm=0.499999, loss=1.00152 +I0917 12:41:54.653079 139813834572992 submission.py:307] 85500) loss = 1.002, grad_norm = 0.500 +I0917 12:45:51.834700 139795531519744 logging_writer.py:48] [86000] global_step=86000, grad_norm=0.499999, loss=0.895189 +I0917 12:45:51.839200 139813834572992 submission.py:307] 86000) loss = 0.895, grad_norm = 0.500 +I0917 12:53:48.452891 139795539912448 logging_writer.py:48] [86500] global_step=86500, grad_norm=0.499999, loss=0.968037 +I0917 12:53:48.456965 139813834572992 submission.py:307] 86500) loss = 0.968, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 12:57:54.216166 139795539912448 logging_writer.py:48] [87000] global_step=87000, grad_norm=0.499999, loss=0.889351 +I0917 12:57:54.223641 139813834572992 submission.py:307] 87000) loss = 0.889, grad_norm = 0.500 +I0917 13:02:13.916339 139813834572992 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 13:02:29.789277 139813834572992 spec.py:346] Evaluating on the validation split. +I0917 13:02:49.636147 139813834572992 spec.py:363] Evaluating on the test split. +I0917 13:02:59.996631 139813834572992 submission_runner.py:516] Time since start: 66530.34s, Step: 87318, {'train/ctc_loss': 0.07378568444344562, 'train/wer': 0.028858572374347455, 'validation/ctc_loss': 0.33931053549452483, 'validation/wer': 0.08834065562690098, 'validation/num_examples': 5348, 'test/ctc_loss': 0.17971988616201046, 'test/wer': 0.051571100684500236, 'test/num_examples': 2472, 'score': 64606.470702409744, 'total_duration': 66530.33806610107, 'accumulated_submission_time': 64606.470702409744, 'accumulated_eval_time': 1793.6775379180908, 'accumulated_logging_time': 8.162017583847046} +I0917 13:03:00.360605 139795539912448 logging_writer.py:48] [87318] accumulated_eval_time=1793.68, accumulated_logging_time=8.16202, accumulated_submission_time=64606.5, global_step=87318, preemption_count=0, score=64606.5, test/ctc_loss=0.17972, test/num_examples=2472, test/wer=0.0515711, total_duration=66530.3, train/ctc_loss=0.0737857, train/wer=0.0288586, validation/ctc_loss=0.339311, validation/num_examples=5348, validation/wer=0.0883407 +I0917 13:06:19.428351 139795531519744 logging_writer.py:48] [87500] global_step=87500, grad_norm=0.5, loss=0.953216 +I0917 13:06:19.432604 139813834572992 submission.py:307] 87500) loss = 0.953, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 13:10:46.145600 139795539912448 logging_writer.py:48] [88000] global_step=88000, grad_norm=0.499999, loss=0.9181 +I0917 13:10:46.153646 139813834572992 submission.py:307] 88000) loss = 0.918, grad_norm = 0.500 +I0917 13:18:11.571276 139795531519744 logging_writer.py:48] [88500] global_step=88500, grad_norm=0.499999, loss=0.908698 +I0917 13:18:11.575626 139813834572992 submission.py:307] 88500) loss = 0.909, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 13:22:52.323980 139795539912448 logging_writer.py:48] [89000] global_step=89000, grad_norm=0.499999, loss=0.964996 +I0917 13:22:52.331476 139813834572992 submission.py:307] 89000) loss = 0.965, grad_norm = 0.500 +I0917 13:30:02.635675 139795531519744 logging_writer.py:48] [89500] global_step=89500, grad_norm=0.5, loss=0.963921 +I0917 13:30:02.639846 139813834572992 submission.py:307] 89500) loss = 0.964, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 13:32:09.217663 139795539912448 logging_writer.py:48] [89611] global_step=89611, preemption_count=0, score=66353.5 +I0917 13:32:10.136237 139813834572992 submission_runner.py:857] Final librispeech_conformer score: 66353.51962351799 +[W917 13:32:22.032322157 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator()) +[W917 13:32:22.032322121 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator()) +[W917 13:32:22.032322198 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator()) +[W917 13:32:22.032322180 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator()) diff --git a/logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..c722f637 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,39 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/ctc_loss,test/num_examples,test/wer,total_duration,train/ctc_loss,train/wer,validation/ctc_loss,validation/num_examples,validation/wer +63.21826100349426,0.0,14.052122116088867,1,0,14.052122116088867,31.06509452222724,2472,1.870249629313672,78.18270134925842,32.10654239086722,1.7203084470202628,30.97378880349608,5348,1.854255781393328 +111.74717354774477,0.0366904735565185,1760.6268198490145,1409,0,1760.6268198490145,4.362626217820504,2472,0.756240732841793,1875.831004858017,4.387652035647058,0.7925199969611131,4.561761838833634,5348,0.7672765895814223 +161.74285888671875,0.0768518447875976,3506.216897964477,3026,0,3506.216897964477,1.2777354587392735,2472,0.3753986147502691,3674.096529006958,1.3805324669399563,0.4144064945354301,1.5873540748380048,5348,0.4231352290831844 +208.47549104690552,0.1181724071502685,5251.343508243561,5304,0,5251.343508243561,0.4732933878798632,2472,0.1583287632279162,5468.964254617691,0.5142548732515267,0.1753546272479623,0.724086608948036,5348,0.2171003717472119 +253.65583658218384,0.1604683399200439,6996.924051761627,7782,0,6996.924051761627,0.3864171428398606,2472,0.1295472548900128,7262.596936225891,0.4007701530663972,0.1409446596989331,0.6215434749849307,5348,0.1870612658716748 +299.8750274181366,0.2235958576202392,8744.24173116684,10121,0,8744.24173116684,0.3434555253564746,2472,0.1161213007535596,9059.876303434372,0.3454691138252321,0.1227004851365871,0.5673220839298272,5348,0.1700477960701009 +347.81264901161194,0.540952205657959,10489.384288549423,12489,0,10489.384288549423,0.3173984007032712,2472,0.1076716836268356,10856.218958854675,0.3102659863434054,0.1111038756661131,0.5330505187142104,5348,0.1594361029305267 +393.4130597114563,0.6633203029632568,12234.653105258942,15002,0,12234.653105258942,0.297861303551842,2472,0.0991814433408486,12650.925751447678,0.2772388136150266,0.1004840512703632,0.502230916512834,5348,0.1494810022691063 +440.97758173942566,1.0245919227600098,13979.592563390732,17340,0,13979.592563390732,0.2780792309181237,2472,0.0940223021144354,14446.687247037888,0.2548133972595564,0.092962806194988,0.475153822819344,5348,0.1422488292376768 +484.9104630947113,1.1978044509887695,15724.52820301056,19706,0,15724.52820301056,0.2691817314826763,2472,0.0881928787601811,16238.72852897644,0.2362978348332719,0.0858377017332508,0.46606361921087,5348,0.1356925602278762 +536.7038400173187,1.4543979167938232,17469.457185268402,22219,0,17469.457185268402,0.2557915026775921,2472,0.0842930554709239,18039.508671998978,0.221420801953187,0.0829181996765756,0.4489882953774864,5348,0.1326896152174962 +583.007815361023,1.736563205718994,19214.5674712658,24558,0,19214.5674712658,0.2476686499935479,2472,0.0815103690614019,19834.014476537704,0.206900756080581,0.0762217953309673,0.4338568253811282,5348,0.1275237773379037 +627.1416909694672,1.819189548492432,20960.067337989807,26938,0,20960.067337989807,0.2418726107329505,2472,0.0804947900798245,21626.68429684639,0.1976364138681518,0.0744093163589793,0.4246268185214487,5348,0.1247236035340124 +675.4134092330933,2.2177481651306152,22705.55812382698,29454,0,22705.55812382698,0.2330298779356732,2472,0.074807547782991,23424.6008489132,0.1849809459075839,0.0696502024115738,0.4099803186454566,5348,0.1196446676000579 +720.7431962490082,2.602026462554932,24452.78587174416,31784,0,24452.78587174416,0.2263144245193238,2472,0.0732841793106249,25220.40988755226,0.1756894884184237,0.0657213557776837,0.4060511360068816,5348,0.117076232317868 +767.1031429767609,2.691215991973877,26198.30634236336,34184,0,26198.30634236336,0.2211009833295696,2472,0.0696484065565779,27015.766971111298,0.1658531643607152,0.0623297409348918,0.3972254793927064,5348,0.1144788297204654 +815.4270529747009,3.0661959648132324,27943.172302246094,36676,0,27943.172302246094,0.2171341195399703,2472,0.0693437328621046,28813.034168481827,0.1571575696981423,0.0598932048318301,0.3931514164374874,5348,0.1118910828948003 +861.4726457595825,3.345543146133423,29688.63876318932,39002,0,29688.63876318932,0.2171851308955416,2472,0.0677391180712124,30607.621863126755,0.1511255163255616,0.0578202498399157,0.3916050698447232,5348,0.1102109786124656 +910.729204416275,3.4501547813415527,31433.99880838394,41409,0,31433.99880838394,0.2121576139992902,2472,0.0670282127841082,32405.74048280716,0.1440294721672113,0.0549007477832405,0.3844594313529736,5348,0.1080673972867281 +958.8653280735016,3.728386402130127,33179.23314833641,43899,0,33179.23314833641,0.2063891848869282,2472,0.0658095180062153,34202.98544931412,0.1363457895334926,0.0523394002539641,0.3755043708873317,5348,0.1064935064935064 +1003.7615249156952,4.018621206283569,34924.75255584717,46223,0,34924.75255584717,0.1997780602780824,2472,0.0622143684114313,35996.56893110275,0.1278055994656996,0.0496261083797306,0.3676198108957454,5348,0.1031332979288369 +1052.7004284858704,4.158437013626099,36669.94425535202,48661,0,36669.94425535202,0.1967958996346538,2472,0.0624174842077468,37794.34977936745,0.1207196938334105,0.046435277135632,0.3633735201457956,5348,0.1023415246463573 +1098.2790446281433,4.471155881881714,38414.99412441254,51140,0,38414.99412441254,0.1932157543430221,2472,0.059980094651961,39588.95394158363,0.1124454156542221,0.0434343763227297,0.3598224439845163,5348,0.0999855163423936 +1144.4815368652344,4.797126770019531,40161.32847809792,53453,0,40161.32847809792,0.1893275246588489,2472,0.0590457619889098,41384.607890844345,0.1056231116995211,0.0410141199709135,0.3530767163718731,5348,0.0979481485057693 +1189.9835596084597,4.889002799987793,41906.71695399284,55898,0,41906.71695399284,0.1878857229700303,2472,0.0572380314017021,43179.3950483799,0.0991121581838182,0.0379209672342873,0.3445919098948915,5348,0.0950803842997151 +1236.2817873954773,5.1664628982543945,43651.70753669739,58362,0,43651.70753669739,0.1841833545712626,2472,0.0560396482034407,44974.58230113983,0.0930589300771203,0.0365426149621767,0.3463119558955571,5348,0.0938541012890455 +1280.783971786499,5.473428726196289,45398.96501159668,60677,0,45398.96501159668,0.1844395076900122,2472,0.0556943513497044,46769.41661047936,0.0892244418318964,0.034686723320201,0.3447293340381379,5348,0.0924926374740501 +1326.9350910186768,5.58700704574585,47144.11619091034,63162,0,47144.11619091034,0.1807859705666494,2472,0.0537647512847074,48564.43762612343,0.0844593807874086,0.0330967342819001,0.3397519458650417,5348,0.0912566986916429 +1374.4993312358856,5.889404773712158,48890.07870817184,65602,0,48890.07870817184,0.1796280329577714,2472,0.0533585196920764,50361.868340969086,0.0803655680552786,0.0315338781623416,0.3392857633394866,5348,0.0898952348766475 +1421.1971504688263,6.198186874389648,50637.22282648087,67911,0,50637.22282648087,0.1773310853724433,2472,0.0523835638697621,52158.77340841293,0.0776614528592333,0.0302640575652004,0.3344761390678371,5348,0.089209675083281 +1467.5598187446594,6.286574602127075,52382.57315540314,70423,0,52382.57315540314,0.1769064359999032,2472,0.0522616943919728,53954.31312060356,0.0758885375197278,0.0294663497541757,0.335301566895218,5348,0.0888813788442041 +1514.0154871940613,6.655657052993774,54128.10344576836,72847,0,54128.10344576836,0.1756454045422285,2472,0.0519570206974996,55750.47462272644,0.0747346069187836,0.029086488891783,0.3327777993614376,5348,0.0880702940182494 +1561.195592880249,6.92978048324585,55875.95197558403,75145,0,55875.95197558403,0.1761567404550293,2472,0.0522210712327097,57548.5589632988,0.0745783904498464,0.0289508242980713,0.3340026475302014,5348,0.0883116883116883 +1608.6839032173157,7.007215261459351,57621.09947395325,77691,0,57621.09947395325,0.1759873620878766,2472,0.0519976438567627,59344.91904091835,0.0744802394846587,0.0288748521255928,0.3337463660965441,5348,0.0884179018008014 +1655.7432885169983,7.346552133560181,59368.39318227768,80087,0,59368.39318227768,0.1755548871096522,2472,0.0512461154103954,61143.36517834664,0.0740967642577247,0.0287391875318811,0.3334598430122312,5348,0.0879061458987109 +1700.4697089195251,7.771591901779175,61113.596465587616,82379,0,61113.596465587616,0.1755761964481579,2472,0.051713281741921,62936.45216083527,0.0736546360733826,0.0287337609481327,0.3340219747165084,5348,0.0877999324095978 +1747.5974349975586,7.8443779945373535,62859.03667855263,84938,0,62859.03667855263,0.1764033919527066,2472,0.051936709117868,64733.03278779984,0.073440406905519,0.0287337609481327,0.3364899435842375,5348,0.0883309998551634 +1793.6775379180908,8.162017583847046,64606.470702409744,87318,0,64606.470702409744,0.17971988616201046,2472,0.051571100684500236,66530.33806610107,0.07378568444344562,0.028858572374347455,0.33931053549452483,5348,0.08834065562690098 diff --git a/logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/trial_1/events.out.tfevents.1789583649.d2c25cf1969c.38.0.v2 b/logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/trial_1/events.out.tfevents.1789583649.d2c25cf1969c.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..10d4aff24314317bd69f8476bc4c1eb6d582786f GIT binary patch literal 99697 zcmbT9c|Z=THX7GXNz>il$<5hymZS4RJzHm2dp!^LIgWbv<^S$xz0h@zyS?k{6~6Cu z|EhgDnlV`Md1{GSxmqU;r8Y|Pj?Qjw1C?r5Q<2k@_gPwV;D|a(ib@J@wlLSGtH~+?JDYa8lob763XYJ(dI{!a`KY#x81A+bvGNyF-D^T}<`UJ(7 zgWpvA&$X4blebT=k48`v%ufx$)=q$8%{ck7@yX5qTp}gM|;oA4_ zP-YB&U!v%Hi}ME4odFeg&4(~&v~7$N=8b}2puPn`PTM}Z05T-+Ve@5Q$H{m239z5Us;d8 zf(3=wCn@fI%?ox?(pIYHZtvz!&A0iQugEuDt?d<~zcTGwOej&z`oPPyS8A(No0B2P ztnA()nO^Vk)y97nRNEe&s;K&dSI|?^Rcgp7Fqynnk*S``l|uc$QuC*&nJdny{`J2_ zq^YE?RMV~e>iIA6EAG!gmQR*}7a><~75VUrHR$aA|8;EV;mRfdKb=j}!f*|?so$#U z;J?Fo{mS9EqFV!Ar>9bPrIt3fwjT369Btg~?X2BAX3n4E=Jx-^WbHm@zWCs#XDJQt z|5tRZy{C<$aSLA5NXbB{2`T#je|H2q?`O+!jEhF~^BwqCZt+K_Ly8*eyj(Y>E=tWw zIY;N&v*$R?2F3DNSWr!!CN}E!SMkB+X|ELDcHkAo%YZj7hWdI}FZcc{q7qn~ulP-m z7wN2|qtw8~)!u%-OZl~DZR_me|N&mFaXp8YrsI<|V`_ypclR_3}iZ2>w}r8h2gPq$x~|1-Y2rsLw{Ah_9*qU$l%A zZD)2Y1`6Byyo5MK1X4^6JLd`%zmjw+#!-u=@NX=jh^pSaIZ&8h;-h7xm>S>W15mWj z;3dQF@9jgfxrR#VJ zaf)1|n0+F(2q-$Wv9F-0O;fyhT@7oL&#(BEKw;Q{kCu_bu6U$1P_)$NCB!M@a5ndJ z7+C@o@#!loC=_XmFIocIxiVw*Dxi3CkB??kP?fe%jBzMXG=0KLh*LB`ic_<#ZGd8X zlcEX=C7R;e4Z(`qQJ`}WC=%89Xc;L~lot&HisJUXggAvdQvB+!^%y8LiaSi_YE&JX zB3M8ERKSQJ1E;7$=}u zvcS7LP#iSpquCVHikiOu*^|*4&D!Pg6XJ`i{}Hqb<-!eW^L% zzi1gLj2lm#02I6G@e<+`f*X{1FZ!wjg^9|Nig75@6bHJ0L<-}LKORCZN?Y*JGEzk3 zw($Up8ydWXIE4$2WAOO48bHzBB&mX;0Zq|dRdB;;J80W#p!hwMkCu@lW|HX=ptvxG zmk_7$Mhcg4zY>9BXnLa=+=^;QQ$+VI#b|vS-RD4|dykKnk)msz2F-y&^)W9YPO%=x z5$*N)E>Pso*-}BFLQ_OIyus+w*O$$KB5eU5&8DDgRMtmJZJ^k&n3oWz@W*kOPH5T) zDCX6BQ9;p&rpUNnh|zb`OFe<&U=u!?O+nQtCx3-Vpm@-lmk_52#&PUBpVky8M(t^0 z!>y>sG{v}vc^Ey(YF#N%tcu~I*%Z|MMTl1m6`*j4<0Zr?B5)kDzdODFin=EjR8Tab zDK?#XiqVUX4O|BljZX8?YzitDwQpI!2MYgGUP7EA8polW^e7rA&VMhgplC`{oST`8 z(eVaJaX^t;z39Ja87Z1{9qtVj3Uzo1af;(Oj=`5=n*qh>odz?x71fMEapNvVdl~ya z0*ZC}_-Gj^zCW0WE2?`8FCk8mh~o%cSl0t6=8kf&plD80c%8Y0(fLX%qJTonl8QC!HF22ngWy`H7RhnYg_6rz2^zEq*K=Im%kCu@l-`eaY{V%;W|@@k!OYf}$l&5wqqDMu&!|w+4z7O+H#i3WbEFYk=Z*CtgCFA`8cn zy)f!3P=tJbR6)^-rkG)R45RfLE>;8zn_ql1n}Vvell;Q7fMQ_H?EeNJP9dn3LDkHk z0fnVrD?4sQsnHaFas(Cb&M}*hK(VbmA1xzA!+rU+fMQI4UP7Eg;31-N*5X}K-rWNg z6s>6rckhFkixwl6eg=x^kNIdeMK$40PSm(_`9Se5pO+A)D8+HSii@&@TxeOnuApc` zQ@k(UkI|>ypTq-&f;At_rl3|-A3ft7py)cAmk_5A+#zUODe?jehs8bZxfP{OQ@AbJ zgA}>pR!%^ny^@cXks@DdTq#gg+rUeRQwZ+=*UpjO4-`#uH&#$+&=f9P!jQt@Sj|~L zp;pL8vni;q*J4|dA5av%<0Zr?1Us$GHrHMX6bGBuo5kfqlcw-Jy$dPS2K^2Nip8z@ zXf_44qP%vet^*23EnY&LqBibPertAcCs3qS?^i+5mZq>=6N(gXYTmyF6i<}+Xc;N` zpIv$bC=yh732}wJ>fMRRlfoVXYn#V`8DX6Ra^xCcAkc;L;yo5MKE2MDq+2#$o_>_}V zL7`1kYz*+hT)cX719Nfr1s~0(pzga%EkEA{irXc;ggAvdQmmb{s2Wh5Sl7XUTTvZp ziYc#FAjS8XgZTWy$d8YfkwS0jW+k8)62wc0Q?x@09o?sYfTD5wxeAI-G{wZro=D-F z(jRwnbno%eYzpe?c0b|f3>3Svc?od}ZKT+DIxrb1Hg*48L7_ua=-zWdidLoz&<*bE z$49d%s1+3y5W5E`x((qa#3^)d{6fGZ>*#gDNo2x4*I@1&?109j#z2e+Fpa^-uN3$uYTuf=_ zrv?=EPk9M(imphpCt&MEpy(Xjd@h#@J(^?TZLLnoU8y&2qqKe+f`b zJjhFkQ}jd%o$h*AqkO}LR#52E6tfylM2hi`-Elu^S|lIMrl3~T@(H0EfWquBFCk9R z2PuZ_&)o~T*mF9$g2I5Nm^sW2DbB1thEFF=FYwW93Mv=DO@CnT&QIed#3@XW!ZN_J zK2TKe^t6J)kfzv|Js&C7?kZUX*F`lWKAKHIT^B9(oSg%?xZ8`D5T`Ij3Wdn+dO#6; zVg5Ysx-grZpd3iA3DGTq;`Uu$LY!h4QoLDH=Lt}R4+yKE z=t5J>+ByU&9!4z1_oJML^U-Vy>bfwgowW!k9-H$L;uIs1!gZ&G4^TW@Ti21xMOT_a zr|Af!==gf&4#-8p7CxFyLFFPi=+H%=ShRze5T`Ilicz~iF9wRf9|u=ZbfYQu?;Va5 zOB_aL0)^8LKAKHIUEQb8WwZy1sWpcDHvn;p@kpV&zw{tb9MFuZ5a~`+G@WjN6l>S) zXaE%RI`PqL3hL_4w7T&ND4dLV32};vNO8z{Y#dNn9s5#2(SxS&$Q+6kyNm5`Z!qu_ zA1x!r5T(M`K(RWRmk_6ziWG`>M&sL$JJOfV=dO#MG{v(Z!HQDpDsKq6IC_tdmXYFq z#GK1O;q{o85T}@d6xko+K0+>9Hric5(Tk>7u&6)g;^g`pn}FhzDj&_JpstHdD2DamCB!LQkm7f*JHLQp zzoDWtx1##e6jL7=VJ=>&-L4NS%C8q6&8DDo(e`5CaiBOmfR_-ba7T&`Ki}i~v0-S&%w4J33x1vmFij96+ zNReoCV>GO&QM!CIn}W*4u$lpRK=HK;FCk9hjkyTQTT%cNjny|+P%u78vj`Sk7Y}SB z@`0k6HXqHVpmO22Rw)E13iWvjaf($)(RsYuBA`gmimIR(Naw=dyDR46_~qTlfFd}T zk7iR)xv2T)=u)8AUC2v_Q}`f7)Y;uzf#P@FE(^F7HHfBo_1OR^CiR|-`%z9!_-Hl- zm5aIw?eV&psK!f(Q>;Y_E5{w%fkN(TLZl8;@0JF zagYn=2fTzh#d@T;c&hX;Q25kKt)Li8Q$*Vcu8Xz%Yjy^Th$ehAn}S+ValylI&-|wv zFCk8`5hTSU#FfL9M9w!RaG`LiY?WAx^O!DV#cO91OW=e&S{Y zg&9p@P+yRXJ+%~cf#P=}AI+wqDs9$`+YUf6H-(oFr`UlME!unHd**XTjdABz)JU4b z*{==e!l;3f22eDaz(>nSal@(mRG=6#otF@&2u6w+g~O8|7riwcD=0?M6gBKLkV5_A z*p5Ikyb~WSBgH7a)mMOGt`RRGP7#U}9fk*M0mYTv^%WGOX^JNCf?R}|%H0KufI>c+ zO+j7Vwar$40E)q-yo5MK7*f=n+%Ov`43e}xxD_>qrZ8yR7;|B_JK7p3j4$%hYznH< zTHHHd6DSs^^Ah3|;Yi_Z@AMfcypv8g0Qj{e+JOPT(H@t*6MKn_Qcb|$~FECMMA(snF znqqJCYor)9;H5X@Lh}M2EhELe7N>DPs&fV}Ax?1!DSmIu!Ckedhx98bCeRdfzrRF^ z_HzC20Y%3+KAKHI`KSTSB4Z&J-_Gz7;uMFG;zviX->{+M$ux!Y%!h(pXurp+yWTH8T1JWopLGrZ#ZGze z{{|pV5swu9FHYi<;wg6w7jr9W3QaLJB?~DA$NjLhZ=L3W}*T#mE8okm7ZU<4L$KGUEAYHU-tYJDBV_54retmX{ExNJNTk zoxAOT;_>_g6%^BGis9Nfk>cURYcqf%z=MyLkz#-Q!@Gb&c?B;aPH_e)48z`~07b@t zOBEE;X^NrsZXreAf{-mhkuaQ(W>Zi#s!e+PCO}a-mX{ExIExe=cJ0FVK!P3SEa9#T zYnsBVhkzo~$|3_O`Y+(4Wu#cR`lKUJxG&}<#3{}r#lrlDEg%<{LS9x-%%CZTe7%9W z@EU#V3Q&CB!$-3zsOutf@@IU9^YuYqLYyK6DfFA(odFbT9@Up}xv-%rJ}*m03O66W z1we6h1s~0(_$u60>)Ah08FCT4mX{ExxPTPr#t*~idZ+)it)Q4mQ<(n{Q0O%7_Y5fV z6x#h4&8DDoQLR}C?$a7H;3dQ0sDc&cl;0Vo|-?pIAiaTrb^AzVjSkB8Igz>o-Ng;bp8L1K>6SFF3qMY z-ZPYhvpIeUuZ-3D+fQ__kdPZ=wfUW*ys<<>9OU~TIFxMRVB={3qNbr1$AJ_wPG4Ho z)eb?-Gj~M+NM9XEI2)u?2(n@4yHEgG+C`!v4zk1)Knh;3uXXNIN@o|Lo0(6h5hF5Ficqk_d4~)Zv*y`6KW|sk2AAdG-4{817WlgX<(3m7SjX z>DyKuQApr}DS+17U!o@E_{_eJrZ}um%|G;j;|_gyN`(ID{LJGa)i=OlkzV^GisA=o z?hW09h&m-}f}lES#Yl9RCusf-9)NHqr)S0X`8(&=PVF0~*eXsUQN<%PJ9YmMWAIli z@bhhJMeQVs7JY`Ma^oCajIrLIheE~_FG}L29HJS#q!@*wyG>Ud@^>Jv-A?aU3`&y- z$$E+=HM|ywj6F>Ls%kg;6sVYTTcTR|F`8ReJi}_JqwIZn&|ke1YuZOBwtOVflj$7I zlwnQ#!sWc|*@dQ16dLA9Bt?sY`f;2-k2MA{aH?weLWxS%PSRA>8GaB7rH-RR53auC zFplH0H#Ecr?t94m8&s8no|15Op)3{_N}Gd=RAEjsR!B5N=fpb+=VZ#`-yr}pZM0f0 z2XdQne$whcDhRS-&ilgva(SX8oDEVW1W_wk`U60Q*hn-)LFB$8h;C!eMF3)bSG{*q zdBwlPIPm6yfii;l97vc8AoZR~!esPpsZ+4x7JL06lm_yhKsF2S3=n4-p-gwH^jN zxcXU%4zmaM@0x;ev(m@(gdVJVNg`3j9^A7)$qRb${d9>I-Gdjkoq~(;^O-AFkg?=@ zl6WaSc+(zx6v~}$Rs=oRAX_3NYY(P|*Q@ByZRo+L^Cha4_h8-g8ffIL4Z<^_2iGW; z=*iTBz3p~k5AJ_{!WAeAZ9YgOMT>%ZU!1-r9h%gD9$fRAM5Ssy_{)`j3!qS{^)=Hz zTVC-GGmfmuJuEn5z0H>B)lgLohD*ZPh4O{4P;P76Xg$n{-*}0J=$xcX#yL6O;OTV$ zDcM&moCAqqoH4Sb_iMaTCx0_WGvRnl63zypdhnFFJKh7x^wSazQ4qN$2y$3%LT3Om zGuC;_fox|S3Ng$68G=kZ7BvJik~~xr&ITzEX2fYtGiw03GESl)3KFpyLG)rj^aYU1 zy)`GCD_>gc8DH@$AIw9L<^>j70Af2>63zypda&c!gINGFceF%96vQM9K{_PO!HP4zmZpQ`ba^b1n_4Ll15~O(Idn9$f2lydw1AWp)xRx(BDIM&e>zlG5!s zWX#4<5-+6(pDH+nLVwb~O@JP3>?RSCwFgtfn_~LyDD>dYOC_q6_uw;&&9MhJJAT9& zdhmqR5P9$9N5|t9D)vs*0JXB%EC+sr%r8`=ZXmoRoPMu z)&&rqE?wO@kQa>4zuj6~#EiVIwcZjyVoW6AY!J#7UEdKs7ePi#G(lFcX&)8uNpx%uFm+ zYlEw0vRPLfKub`Ts7Wyszc%;BVOh<(8w+M4x`jmOpUgzu)92b?CU$5@6vfTNqO8j} zvX1>eyMmc$*HNOwG!u%^@+e_l_vuzJ6Do!hi7J|jMK04Gf|-!-CDEeIgxvwbRW;~r z$}Py)F%wC=6f<%4Q$DWhvmVAVU?%1bl?cgdCaB>>O-)w-GjV0KM744=@$$fOG!wfs z4fViGXibpl$z&#Ok8Ow9Jojgk9~6at(Sl;33OhN;!~t#wU`^ew;&)9+&2f0FYvRM+&26-;bNCzd;J^(U0NTMMM5>XdH zTz+*N4W1^-Y>?MNkX*M~?EplxxkN)0#HBHU zoVcbt89?;Ek1yju{25=3j8+kV4DNfp6Mzhox8%dwAk@yprHdt4HwM>}Xo!NCv_cT= zAAMIs{XWvjQzNx}X>AxI%}k8XJC0>6%z1_?AZ<~X2ysZ1nXnu21MjQaEx6Md%)}yX ziAH5+qGq3lxLTZShtC1DD+UrZDQ2R&h906F(v6!BX5w;hiO@fpiIJDBwt|`HJV>G_ zZYE|wI))>gp!wngn2B2>BsxqpQSHtfobwL_SC)gB=wTs|sG^zZIsQlhn2A4ABwDnY z_!?0g7vtcNs}Yc~BwINkMAbC+m{ zg5>HTh_hjKdjL`Eo2P!EyyBNIJ~w#r-~@t9Gk-c1Kxz$_gv$srvd^9K0J3VlL_-wh zcozgQ^eamMki~6^W^*99jQ7KDjy{DTc5g%507zg*NjMvX>cO^6MhpOuLL-TWD2TT) zf;5QT)f38I^hU3I4kXP+S`XHL9FAB2k*i6&0qN}yi4ccG^??#GT;r}pi|)Z2 zU$(`?So3Gs_OKXLvL*3SdT?$;M-)1(eYGd_VEsIakgPqJ8s5)0PGg}5n-xh^EAPQI z3y)wA{yjz48hUVEsYFkv9-OsQ3D>;B@$i076cWBlBt?rtyFoa8u4^sVK@YzBN1{@- z9vosZ=o%Et0jphKaP=jV@xj2-=%dK9uGWbqP*pZMa5e~aL$C8T<}H9IXh}3g zLDbDKxy6gZ?gL2l{Pb`RslLjCYw@Nfb zK}yFV$eOdaj{=BeyWV*m$np6!NR9I+5#+(uoL2ziWhe<}gHW!hcT~c50NL4Fq9F>B zG8sYo9U0OOKsLnbT3sq%T9FHAkfA?f5F}>LDO_5rsgiIu2({_eYvU4pyEZ>Vq9F?6 zKNCSLoVGp$5XT#aK^#c12Mv4?Q-sqeM-LE85(!1)`};(*Fgn=wug(&_B7Nz3#myfh+oAEKwA9 zMMGBxRQGCo`6nf_8UIwnHnVm#PR#!v~Z(U%sr{Ie8=Sx&8cSX+N0CL+}63zy(o&N?w(woSi1CU$? ziH0bM+!6%oUR3iVT%j z#a}_;^+}2mzX=ghmO^i2X%*78Hn40y?NIkh`RXe4VX&BW60kViOz;Pm(u+hqn}vFv z^l+VR-+<+51|cHKa@$A1a@l=&@4r)`5?GwCcwqoR88aNG0P7wWYfGo%0C;f%HSv{9^`O=mY zocQNqdpq>~JMlAyzb{cdtDqrS2UYCI&&+?v8U7X9wR&}t;&5d`jBd%+P6EfCGokuB z$eU#|LO@DSPB~+P>)SQC&B>vEhjQlPkDZDOS`$jLHf3sn@*O6&8t_+E?%;(O#eQ0Z zY~@{9V|fmCX z-H$P@=Lal16bDNNA<7c58Ck~lYU>6pcM3}eT`g~Em;4zlcHaAt<)-_n2wSoMk^n5D))FG3EH2xTrJ$&B9FtTLqSda-tJu+3h-L>*H_B}+Z)lmMr04G;v)lvN2pwJtH z5=TW@jU(yDu;b0kKadPoW7mB`q%y1VV)t7d)3D^+dO)Z1gpiYBHGY3kLZz>zsy)GK zm=zF8|710qHyrE%R-?{4LQ>pn7~I`}WApgjz5uMo=Q2WwX*Cvw$K%9r^)J2zR>SxY zp;1Mv5qbamJFprG3LPbjinbbRu^zZ8%O0zYg1i;gBLt*ajp8%QQK^mbBrmWUvlz`$*NTv>+-hue|A1EGcAeG9U^QIY5rQ&Vjp(r(y38Q z#IsB1`Qlu>UZynaFdpV*+($x0bY4uta9)hAPUs0NDT~`AaxB{;7%b7(_hWi2&wI!L z%iz^SJe!4T&lhJ-Y6>jFHWMPEEZPys65vp`F0gD34V6nPZv*Qi87zfQ1rJE(nYO^g zB~_z{co|tXeqVSOSgcMEBBCq}B9Z0PrPx_v-BV0kA;`$ z6Y*@8{YO3_i-YHyBw#V>Lx_m7H0YlLgPu0Lv{yLPV4$_!P3ZxL79uOTmkpM(O2iEG3D-B0q31 zvSdZK@&=ZoQX-zsLVc|{=zwnxU@`tlh={UyCn1Z~ANdGi*|h)BCXOXOnZZ)DO2AUD zY1}ekxqqC9XS2K%<|S1x`zNrxKTU{;vN$9o%fjz+U7@HK7r%?*Sl<4hrJ|_2+RSm% zw{^F*c64@g8>m#fnu?sJeEJg$T=Xr9EG&Vl?Po%XqoS;au4))|yx7riO~Gme{USsv zvl>e^pW>M29CLgPbOAL>1$3S^j&tm6+~+tu=`XZ*{f{m3S70^jH76v+twwg(avYn(mFlTrH3q2@LQJbs^fn47-gKa123UVvLf%q)5CT%HhQg;%RC+PX;W}82{Cpij|G zTd*2QLkZc+t%i3>L$n$LwwacJ)p$Ib5R}Pk#1DOnHM868m`o@UdJ_mu(Grm(C=o~c zD|ZH~(S8~sQ#Gq`F|2d~l+Jc_V?J=@Ci)_ybiUmq*a=9^JP;2>rgd{7o?SYrod6g4 z{@Bunv?WAD=f(UYF0J;To$4ctqN-U&c}oks%3vAXBplOoe8mz&VDV~1#Isqb_T0Sl zAl#vO*OCwsW$AepS=6-*wgZde<$5t3i+?(U_7A#Q zuwGl|5F(;1>gmX`?_`7$uo%6X@snfmxy4|KT_CWtd*==40W7xfiFh^(b;{If2agTF za^gE7BFa+x7P2%OK2r?}VZjf***D79*qOUh+jAG25!fWW-x=Y4V2#@P5+#m`YR^Ts z!Pq6zKDN<<_FTUKAyQd;KJM@c$FzInFdw9AM#xFo325g02M3m4&v!Dk=Z>0$(m%Cl zmxL=bp*;`oL`aIaXZIruk+M;^%G92_J}$(XDc^Fx1(b-f^9W7R65)Oqr%=;i z2A<#b#g&k$T6@+$X?GP$=W6|}PF%V9`u}pHq0~l6p1cRxpRfaw1{})A-?{nOi->2J z&X2-7o2Hs>U144_2N5Eo^HM69m+!NOp8%HrI>-bRRsvP3^cmeFc!cLB?>K_j1WEZ6cGEJnk(B1_-y>mLKlBy%F3 z%|d;g(d?MkL|}24LWqd6_~#?buKV}LLQywOjn=+ZzQ(q^l4>>f&+Cf~Z|tU`pFnm0 z2BE}JQC8zpJJgJbIFbp8O)1wSF=q*#qXwLjy)IyC7L z2UcTZ0ipCyR-+*E(i^ZEz1|U$;#OnkZYQMdaB|>5uo`2(5JF6=5!W>Y`6k@`I0~%B zfIb?+_?o-V`W=HP$sLf<6S*GyZ-#>4=bQV zyzE41ik660uP_7+JMAJ*v+3G*_hm=F=2mqmhk34iH_ZM$9d4v zz8VzFi4%!6K|KdwBQd6me79N zk;Tk;TQTG%%A1I1vrq>ybp0N49avm95h9{21|N{+L+#ipz+yh`b_mC^xs1VLdMyZ9 zqT`J}153&@BA(6CD_HP;LwAc^$-pve79k?a(!2~=GJ4;*3q@Teazhcv68%-G)%Z{# zI52vl-ugX2)%h5q#8FXJ<8|;_?08Xe+wX$an0$&5smyA;dwC0+@V&%0p+Gk&nUIrW zH4a^VhXb=4oADH^#+Woh>7T4dVD(|0z-olwAtc4E#@F++kuv&E3v;jLok(5Rx-Xujg-Ww07XMT8h_H3Ad```kXiOF%JufV)?ZAgfS&Wqf4oR`*`inD>`vg-RO9LuL) z43>EV1#RHVkcrt)Fc-Ba;@K=3Hv~6R`f*QvfF->%AtK6B_zPKPCr{h~EYX`{?s6=J zE34BM@Yn4CY-v6huAyy82_oXzEY#g|(8J~9faTy`LPVTJZYZ3C*>Y~tR$$S#ouz!Y ze2wL-VX&-iDA;W}o$;hDZiWn<&mojJD#~gs8s&u@@3g|R(_l5Wy&yy?vl?w}(s4|^ z6ZZxJ-IEeRPKwpYbt}e!)gN*o9IVEXGD7K}tcJ?fRqMcNT>ni-id&754uY3heI8$W z3s$4Cf}13VOsjEkpI|$w!N3uZz-lBY6B<>t8oO#IC4kk~-joodt;Vjy8CdyhtUo^& z^5&sN2uQIS8)5;&L zM$N8-piEXHzmG0jjnHYc@pY~yeF#m_60yD}mWU3ny$*rZ7&nNJshZU|x3g_Jv;q0+ zs`t5abIF%cI%^gPTAJ2eQ#>bNZzd7XE}hhK^o1r?hoE$tJS9X#=jBo@oR@JPS}lRa z_|WfB_sUyZ;#LOB&GG)&2G%H^e+~unQalmQW}#l$(`X#G7+89qBSb`55^Ez%m)>ns zfW>{r=uI5U;XnqvsRcu|$M0Sl;gv>ynbL_%s#q?0E=T0AtK7My*{#p z9eX|xin>$sxp^GRrEsZMqxg+F4#<8?7Vg!oO(&E%D#~hXALoG`{cyJ(^TBFl+#^IP zvl{Kp(r`>#$NcUAUFl;&PKwp&(BUNxY}=AX%3w9*UlB_GWHs`vVrzlb82FZu6t@}< z69vEY^-%ZieXtsH%LpN+)o3a&uo^>hbo{|;-2F{xRMBeeymEdfSPesk^O8kHTaAgM z1&>=7xUGB+c^jon2uQIS4OR2dYP?HW^arfQtfqvLtX6{>V6SxjBVaYwwkBjNw;Gm@ z&CqHbu+Yl{tD)S15R}PkZ0Tc)R-?b+V_zr{&iaI=Xo)aY#u8zbYP}GwhGq{!rfOCr zeON61)>`4J&F8psvwjbwbQ)~hgr)OWvdIuAoojuGcy{Teo})Lnp7;airA`nbB04YY z8{oVYCMMp7Hn1i{RsMc?8~7Q?V3~PA@ZQPq!eIk|MKy|uXR}1^x`Gw%UgmZDKs5gZ zAtK7MOa)oa-Pn&C^;2tRJ8>+f(F~R@lLakpK(PD{U`bad;@K?JZd1XUX1^dW1(yd?VR|qVzK7@!U zi(OM>nN(o-1X!}KDycsxUt?Lv87vw#1Qsw-tvT9R=X*pvo8^h{-KOVyS@_k)(i}oW zlx1RbWJyd)h=ZbTV>D5hW6AzMOGQyhgc3(ZS&cWDj@Z#7 zUwo+tRwLPz5UI>+czR#MF}<(ZJ`(7ZMiFvStcF=a9u92t&6BUeY7Ck{DE*Vwci9G4pJx$5OsjG0%tqu(s@o+Htj0uVLZgaS!*Naa7_b^s z7ZPH$)foD|01ZO!_k#x^Z*{x~0V!6aUz>NRaJ4{;_A! z2JFWYBBJv$QWZ;QgxVVwV9_xQdB(A%Br;g^&Iqi=M6I$AD42GAhS?t35Jefn~J!PoszBEiL{mgT<|zFXqK~?}z2UGTx7fXR}Z(?QJc) zBFIa~PC`VKrE6d5lLzw2TsgvXy3zu{O?FG_9Cp355HLJ2&bY7SKH?+7K1ifYe0pUuE7 zv1Ha&9ca&UzYrpowdbb~FX5OhLQms!a$~vI0y_3bl?%o_!GRS$Y?TV_*;kQJ`lt3h zr-!^6v}Y9+LQ=dvXZ4?il={D~mB3DbpDH25Y|l-HY(T!Wwl^Z7JvV4eXjHL1XPaxT zhxVM?i4ddP^Pg>>v8n!E=z<5B*>oWUq_pRyI)VqcTbJs@Kzp|9O(@CQo~Z$DwLDw| z?YYK4Lbmeu9COqb+w+oFrWc?+j~zh>%G91G+?tB*x$WKQW>6wNjU_ZiOGK&$mWYi0 znJ1t<51mBFRINQfPkCqwr8CGQ?kiVr!Y(mNr|Ss8I{`_LCg3J&#tI^yT{@|a`ji4Y z7nqlY>j)9id1E;e(W2L;nMo``3&XpjGb z*Z!6lkMN-7;B$nCC`*HO$a3&&GM;&0*vvlz_MJOh-b4< zdGYl(uMI5DI)sQQOG!s$G5a_p0<1>!>Bh%6mLCtKS`FjyIykw7O}pURG}Y!2N*ons zH6Gebz>cSS%IhXrjZSWaNM%-|)xZ=S({x284WJvbl#r9M6JXjR8wa-H?8awcHJYv= zl>W(T4AC1a2Ua6&Ga)H%HI7>ePR?i(f3OU!#7tOLPn+>0c{Xsgj@XEj`v7US1jK;E_-CIqBdjh~@{hw~{#-#3EQNQonqWVIU9 z0Jny#F9xeom_*1{ZZ&3}az?B1F8PusSPkP#grH1TBY&3@T8+tPA{C%SSY;5Jq9tOf z4kqKE^bI_rpA3kA9BlMx88RnVU?+ zvsrX56yq8TI`|aV*yS`rM3m*MA+jv|(Xa^e5)kmE)8q0rcD#VWvP^FUvNSh0HU^e) z;Y2)}B}e$sAwbzv5%SU`nh+6XIo<_X{1S>)ps3%j7v#*b_!UXD8jGI^4pRzPHQoxS zR<|IOI4a6&B%7LHM<4NfyBb)H8ybX2Wme-*&$F2D`bSDS0NtC8gq#$s;oIXL4s7Zp z;}KvrPIe)b{>f^X2fdyGR%5;~At`P(b_a|^%0$PaIbb!G4I+e?R%1ox3gipdj`<8$ zBi)S9sG`*{I5{Z~tcKq>LX5T=sg8m-yY@}k@&@u2FqsgLVl@K71n(lfPw3Mgtj0?l zLP=JuK@Cu~)iXJ;8u#W9vXxtnbEQkrYRn!!c_mm4b2mazCabYI*c+{e+m3a(A$@8o zp($D-{JLU^_#So20j!44Dnh1eRzsnY|3)aClXBelapflRHKTNH8?_2cr?>0R-B4tT z3yFAk>7>r0=z1jq->9GPkq{A`m&k57FL@t6`vQyd1FaH{CHNhKrIx(FHhInIji(>} zctymsSpt&^@!D_p>4+S(fwFgmh$u^N4`k`fu-APLPV5hb1!7c8e!HJ^3rnGmvD~7`wN4`rsir~V|D5u z?hGtJ(L_9(h1xk&O?#9NEbg&{h$xGBSJr^JSor1r|%3F?_Gj5<}=oy z6|hW;BI0FaIrb&s1+XZbAVfr2-1{O+J8yj*gfj^~MjpxQ+uvu9?i`W2E z_uCUn92I3XUK{m85uc0&Az(H3bS6Y9vl`9op1}*T=Eep4f$niPLQaa+a1OqM19R%> z)Bvo8MSnu+pRC5z)A@_RYWNH#B*m@96QAKodFWmn1+W_T#t=eGtFh#u;B%R&cW!Ed z)i`cNXjIW^tnZ+81+2yzYeI~+8Xx)!-rm!!W84e!7BZU5Lv6@bZh|Y^cf1DTNRP(XGQZH}&BaS6!RSo*1N+WZ@>xZ*$ zH%bDQc5jGyHVd_@vBhf=9&Oh4D#waGa?Ot)4}N?AtK6RJ_uPRJpI}gSZJjSZ3G?__0yRU5oIwQj4Vs@eDDL&QLDyfa4d;i7%UokKA4xE zqjod{mTjAecs2|5;qJ#zdfsro>hUy&ET8q!{+yf*w5 zHsf|036p{9!2v>vqoS-v|3X6)d9oti60AnzF+!v=s}a>Z5u32_+dMO%(@iAgq*#q^ zmoso+ZB!o@fz`0QKq&o_)v#SxhQC~WIgOAMw;GeP2P0+29hXDFYD~F92r;e3@gsuI zW!~4)-33;|`4OQ}MXT}9CMOoGMrke~Mq7>F^#w00s}&4b3waw@LE4LbNhiyQsk#@E0K3I(fa%Uyip-fgIp^+b2 z4LO%MJd2{6BB3c-A}$F^M2*XCOTcRMY(U6V&1$F)dpZr;z*H-(vCqm|+Tj33=^S5O z@D_T=D>Zx}?367L&n}(RvuC}g`NN=eo^U2aMCV0!IF`=Y;U)M}SSOxjZRS`ab}(4} z^b@qSs3k3YpkR)8O~kWVioygxAEl9XrxaNFd?G|dS=7yt#ohGtZ^(=BfsE?8<$2j2 z%3!e^w**^S(3VDPktLRhXS0+DH|j^E{KSVr-;xLsQI^J|kVVlu6<^nV=j=LwV_6^0 zU`egM9$9uNTi653UN0h^%|d;I-0X71R>;f9Erf_DOYJepvTeXkY|qtK?1<%9ZtRiT zp5K~hVFPPdt2h2wS+$LX5=TX~XUE?iP-NPZdM41G7Y7g`m9^)^SK_g8Wvb+TfHGee zO2|oR&q^WJaA20kKYK%aHrY=o{Zo6^Ix#~Db^`Q{5R&5UxzV!$NNIFQ{|mI|H75xn zW_$knXaVx6bQ`=4+H;3LFA-vNdmf)Bc=6ih>60Olx83Q4fRy$e z7A<%WFgZa!AKG*BT|!CL_Dl`1YrVUVpgjj?6S9@J=Q+xL*q(cLS=JERb96o-C{ufG zuqp)GbIOM|xLNq4n9vk05jV`SM0{STz5&|vq7Q^j)!Ose39}|b8yMtfoXnM*pZ_m6 z)Txwa^#yk}>rzu5Lh0DbZ&6OOxaEH4f+ zSYkE_HtH?p0v|zMKBW-xY!>Py$3x~P{DI}mbwWgx<;8epS<*w(bijlrRfGDp3O39kKn^yyX*w~4U|Cvgor3h#zbUM8>RUkiu&}V@opST zcDz)pG4{uGY*yZ76D9)HpeuwDM@3l;d5e}P()E><2Urb9>RSdFar zgb>qe%y*rOd^>t*t^%vE?i-;|MXPbmFMJ7Djc7Sb$)cjIMq+GRT$Q)2b-KZ-Y+RcV zkYY9LW3*8zEY7h3SdBf(gp#aQgBsw0vsZJ#YS=U*WGlBCi<7sb)o9Uh{3oy)iERi$ znXE>FYZzLM(NC0yLb+4cCNxD$#N^31g)MtqPXw!RUZ0Sun$>We<9ixPXHLf`Z?4>= zoMe>FO)C~-=?vGHRRl#wvkMW=E}hg)K+&xI-(X%6`w$|c^O7Q%7pM7MGJ)ms*T)4M zOZ;gD%hxP{ZR)EqKnqw>tLI4K*(}s^^kvBb>cEm*hY%5EiJyu&e!Q?z7O>d6^zQk* zyro5-W3a5X7wl>btLNwpEX$S?@oW|l(1Q_8{WfEYwGpA8#kdLS8=fCqzV9!e$_g#n)jyAusn$BHnT= z{+Agnn&E;qR$@Iv1z2=O5%Fvm$^xFyd;KM@3nUgdGPjVfA=ZsC4rU^P1B5n{B}IJvqLuFA$qc}*a1V~Pj? zDOMx!sNfauylRJ^fz=2vC6r{f8q@%L9@#Y-tVYwXgly$jLw-XjT8)Jc2UmdASRvO| zavjQKH4a(qN2~F*Mj_sfbgWHiik67)wwR2Qjk_!dt1(lVkg1y0$lmnb0!nAR`q~Lx zx$#M3l+FYVLFuf!uwfr4GVePP@$Aw`ZEmaW*oo%^OzK96h|Y_T9nOnhuhE{sqSEEa zF^=VH27|?9iQw)z^2(zEU}-m)h-b4p%zL>KEui`1`uMTy#{NXSj4U;L548oBs9}VND2t;5vi$$&sPSO(+jS4l z;#hJs87wFA1-7Xy_!53p*+i9yXS2K(K0s-+;0PYmR;ERWh_cL>i!4)vf1iUjW_-Ej zX^tf`i@_40eZTgv%gCB^t$tFZZS;jje%gX#7DNxjV?oI#1 zv8>6KYBfeDC1A61So|G-5HIKmp~O*9R^w2k`Y2-5C&wMEM&ATNq%x~9&0Dam(RRS1 zLZF*5tsyWw%j+o^Yi zfE24S??iW8-!uAOQ3R`Tp^Q+H)oM@!e0grYBUlZ;UxaMsR-OvJNUsGWeKSZn-QuiiTd5mA;tF31uT5{2h# zY^iRR%ds4P!C(o{5xn9$sPDZMV4HmE6Y*>o>h3u+rur;cul-dC5m6QcS7g!C7%>m> zvfbgRPC@w^i!5ZYOgt-KX}C2^16YFGh+eEoO? z@As9y?H&XzYqSm_C&g-{zc_;f>-Dr`GguA9ZiLc5S&ftSpYDRyc-4oH6t^0m+w?-p z+n!S%g4K9qN(eEnhEva}$ftF!Z8BJmrlSarDq4*wJtb?f8gncOG1_Xh*6xX`a&o=u z;gGkosf2(OtKkwa_$q11`YAubYP7Q>lw`FU)BrDph7JL%(PKU#Te;OJd9oj^#!$O@ zF<>>Gx)XvjS&fU=V$o_CPkE>XCF034LQ}LvoDr0Wb3?~@fYng(A!Mp%HQuzp{0d6v z`eEL0xpK3)luPkUkMT)a0*4T_A-cp{!%I)9H9{32qg%}zXC$#)tdB04X17vUmZ z@~l2ujm}yZrxljBH1AIgmY*90E$zuWH*C+}42gI)3-w0*Q@f$BAjfHa2@z2ixh2SQ zQoVRBPv^W^Y`xumfdoXC4&%U`Lq;S zN@}!G0+!L*w;C0d=cVumgJosDU?;%gLVkBxW2Ie)cs2`_msX7?;m6sD{Rk0Jmcr%8 zQaxuPezh?<%V+|}lJkebQu2Hb=H+_hqWZux@i`ICW}#jSE&Lg82YG2;LWqd67T5IdH;!xz-m11N=S-Z zjX9+~&}z7~UVRv>Mo=F@h-ozzZkddHruMG-U^Ns56B<>t8Vw(Y9RaKH!i*53tw!VT zf^QOcPA*;ztI}^gAt1$SJUwlS>pQv2hLK=3K29c-WVIU90E4F;Nd>FnYfH#hZZ&?~ zib1PUt<$$8uo_3^5rQ&VjcT4}(Q34+sih1hVzE1+DOw^lym1P9X!-?%)mXEXkg1y0 zs5^fazUCJE?MEh8Zf>uZr(X*l;XWTr=e`FPkx)AC)cPrjXO~Xu?)m!Cn|S7dmkJ>w zIxmHSd2tTxjPIQc34Pb3xV)uZTE}3Sn<%*B?4vMq0m6+qh7fY{0do~!kbrZ*uxQW4X{=i(!i)o7uPl08S6%o&7p>~^8?6x-smUc4< z5mA<$RmfuKa-{~aeB8R}8pm?jkHK=ycOJ4NrW{=iER%N;@oX0AA)VE1L$rX=QG|#n z%Y)U(ax7l61PWntn0n9G2F-?Q#AvwC9VZghmzHbKA{@0nnaL z{vpKZ_T1}+;In1TS%+Rh-s0+L2v?=EyN#o@orkN9`y6Mdfl9TjsmN){UpX)MS}f#Y zaaU;1Hq8hnS=%!;z>HTB+n_z4YEQ^k-k!%qoxt|oKd-wDv}emMgrH3Ad9&3OY|j^K z49tOYS7QL7DOMu9_xWIn$guj-7TWV*GeV|n?RjX|*0>WeuvYFUuG~ayW0X!OJHefe zs{aZ6?SP97hhRjq5H=G=Ec4xAtE*}a<>KZlH#~68QMUOS=uI!WqS~VWlO|l zYy-Pr)~^i(Q^SggXR}aW2Sf&E<^hY<3_?Vd<@y@T@v!Q9TLH_%?;15r%G<#DoeY+R z$%4}l59By!0!v~|O-VePg?cAo{H^XKz+#|Gh={UWT8k`O?T`NfmRdfuO*odH;S3i4 z;R`S?bE*#w0+uCyL_C{?+6frY`Cb69910~wL|M+SLl%!UWwn6CrrX;Dj-_-jgJobp z!SA`eu~Wk@P%=!2cs2|5W91(ktULiMwMGykqAZE)ktM0mbR{V2?WZ*P#<2uNNwpeo z^LC=u@Tzxt6=dk%EJBH+qO8VT+xKWSG(Qg50aioKnGmVWYD66tSdGR%qv``)iYFl_ z#cC{25Lk_icbwe8YWR8+O8;av?m6Bp0IT7(o{$u`8f97nt5NIS{C;3H^!y1Srq!6* zKwve7u9<%WtVU)qp;1Mv(esLf0a%Uo5ri0RHMWJ1#8v6JNTDy}%_f=QebPS|H}>i zf#^|pYy;WHO7N&Cw^>9yyL3`_X&ZcZ;D@u%oCy)Jd6A11%!}Jo1$>TvG|9=6W63(i zVA+!Dh%6Zfm%Bm1ynLC6XR}bYX|!PywgHEmgor51;f7Bz5L0;TG&pyMkWE^F% zEcFv?)Q>JYioYNlU(G=h&t{?8^TzSr_QHD2s7r{5vP5k{mJU~TKL?gQ4m~@(Enj0P zu?&_)N&>6Vv0?ACm={+fp3Op?usQ06GM?YiZ5bgV$`Y{|S^Uf=)B%>3?|Zp$Eb$2p zmU{IC);vRhNljpx^oxjRv(URujbe2mFMn$HmJC9aCDa#LI@XDFgQA`s+Tsky;&)oA z)$phvf>z^+;atp6WE7#qQBhXo`jEG1HC)C!+ykqjaf}eD%xYxY9K!qkKTqnZ0o~S9 zgq#$s(Per((#dbXvhlHyjwE~g7p>YM7Sg4HOyLkKag z#`D|bk?)04$|$3@sDq4-zoi`_f)v(DY#AvIb>SBSb^7(r+W60Z@VnRTQ)mU<6 z5-QaQUbY>qhSCQ@Nmi>t4Y1DOleu6ubiNa^m0JxjlXGY_Mt#&B09GTlnyTbFl*wwe z`;m!Oqxp|FUQi;wD-xPwCBl2?7M#LeN2V)*)u`2wkg1y0$cWZ9fYSLwcXB>gZlcdH zN@tY+3~bM)t7=|?(s{Ed5zj82R7>l7+3*O=OUwX5L~LHHy%|clXXZ4;pfn`QNLPV5hSpc$ZoO2)yin{sy zfTbMEhIFY`W6RlKY*u@fvql5erA34iM@3nUp`G5K)o8d%R}rj6&y|EoWmdz!x?oph zSP9%-{dP@H3FN zZ)XSrDOSTTV+OA8A6?(S2diO!fl!jwYET1wkvIwuz|K!2WGlBC1GO%q)mXj1;~}sb zZ|@L-GFgp_DNoR9=o{!yg%a`W5uqtoBD^mNN`&X-+o!;4EXgBes%ACx4tHDurSrvX z^+Q~_DY?NYoq;Qyv2=zg*yTf!aac&ivr8v+X!N7FnLS}%>iZBPV)G*Bxeb?=Uy^bv zusAL~SL=Ow8_2!GU~yb4U>Q&nj^_lLt|j8xEYt&(=59;x0gIL&AtK7+vK?7G_f~5M zEIlq}+HfqH_ZcjG-wAd#ylu*ULtakZCF0pE)cXw+bOk?(J@PprBFZv15Lu2Cw3!4f zHe=#WaV*y!F<5%u5bT^?YIh$WpvX@q;@K=8gjOT$j&U61Wr8gsBFbX716lgGzc>pl zPG4uX_)xya&OTwVXxDSX>*ZCWR@)#ii8XB{@obhdAxlxD`XgZJqe6&?vP=&`mUl(7 z)uE`jusYL+WBKt^s?}(*L0~nU6m%N`)ti}w5=TW@jZYg(&}w|V>2nUOM*Vq&NM%;z zX0BjY^qUAO1z-ruDO-PDcjoRG> zR-;~v{pql)@pBU)#Izbqe+YIp4kdj*16D(0JE2iUtKm5`uNGL1(P4xbZ8cgnvB6c@ zY|fTikhczzgn$&Q@%r{`RC-eDavWHVpcq0)R;xh`aJj;T&akVoA&!u(+-f{;l7?2} zMP45ruo^Ft2tk>wMl0i7v>NR~#$SRG;dGJE6e|(l=L98UjQNGlU^SLsCuFK-HKJ}! z=?tZFR&vM1T)8=($0(f!&JI{QYY%B&2u0>y8WGPfom5Nfxk&9R%**Y2goxO@$W08! zrDf**81J5+ZnJ*Mu|&RPune*kwC7c4?9+f{^KK%Z%|d;=H}v_>!2fl2CQeaYc^tQl zlSRZ}QZ6d6ft6E0DRdGrF^H%EqhQb==yC*!iL(lcjMuIxEb9>gBZU^fMt6S36W%RIf5*A3TG5R zQUA`>Z&+Zd{V&V^ih5X(BGf*3b8z6skCn>*FTk5nmWa)`*84Wj&=pUTBv6S|BP_HH zRYU1Bh%boh`H+y&RKw*WqZ(6RSe*-We+wjWYE;8ahqNUV}DNjfWe@;gZ3s(L_}_2O2pxE4;xU8;ZhQ# zooX2SwmgN>*{t96Uhhat>+R)~PP5;b(pfxyp%Jv_*~Vl%zjTUJLnhyr1En+AiiAj= zmoaaoVd*qasrdw0u0~p=3oNYz9F|rU^LBt*)=kr&m!7a!)wVYe4)9yo`guU z%*sHP8O_%30L!oWrd|Te%}AMeJwHiQ4m|V5U7gZ(D8& zcLE%ONXTgIxpN%z&|z)lzH*@ZyMn~2Y0t`=XOON?H)|8L=N=_Vdam{y^e59Z(4J@R zAd%ATd446+o?CLBehBS(aTE#Swr9_(mB`oq=7~gT&%OIehL-L5#mOf4f$x&fNtn1j zC!c4Us;gpaBg~s|3JFlto;TNR!RysKn-epR}psnE}XwP%XNQ`#v z`Nvzu*q(!bYP=zoo6Ea7rSq0N9RJzOj`#clii~3u8P6}BqNS8y9+-*C52IU2h}3x* zGyN!DTHRBh#XuV{U)b`>A0sWT^ivMY#BgSro8f1P*MX(ogN*01h}LRYZ~N95=2+oF zLL^y?k0Hym)0go36t?A^`vjK!SPsj_!_1w4C0-M7)99k>WIUgxi&YH=@2_isCAN-) zNU|7YAxrkT!7!MYNEIZ&L~Ll9Wxx^I1f1I#j1ubO6hqViF?B zGU+(7JUE(P0xZkk4RProxyIt-IV=VV%)S~0M$2*AtAF{C@q8B18XATB+!A21Rge%# z7TpubvM8*0A{6zhW@QlqOJ$;3)#&($QH=sKD}R`wxLG7gpc-8@-Vdtp1JyYHItdv~ zH5Bgo_~c&}U>yW>uFfP*jcT;qW9|eP+*bL4YP{t}lAcR7TGqc-4XP2eoJ2~i#*{4P zp+oZ7U2lVGWUL}VT-8Xp$UJm#{H13psK(9!lA)z)q!i_(!<_(+4@sCp z8B{~Qn*?g48WyrAs2ZjR2k@(#QL!XbvP9fvO2ppUq3fU;n+}l}?NlS<`+ZZObY3`i zJX$C>pC)liXG0IO=fM?&pK+sQoqRH$UphsqF~jVDBb3euXGw_Uc^Q|DmsaWQN=IP1 zdL(b$z(^YyO69OrIWSvAy>u%Jccly|AmjNgqHTQMbu=6V7WtPXM3QBgVcFH;W(6#B zll$fiEWc-PSWIl0rIh83a@_rJP8J!@XA%8-Hq{Nl57I~HlMqRkfgGHd1C3h>f#v7v ze^Ci6cd|Gv?p4h4!)fMK_fu_A`h_G!lBGQlS!{K^@UvbO!Lrc|go7A}%RAWsV2@|Wv_5el@s=I!C4D*(mO#;-Y zhHW>q2TtEa3r|pu#6ps!scMK0P#&tnyBa4|BzAPwD15CMRl}~se+#IFtegaDq#But zV;8{2F6r{6ze2f-swA0`B_ie|mWX*CvN@m{9Y2v6?Nnn@fc`otoef|75+;vV*i}s%tSjvhx zEJt;ihYqf*<+!)rj!R@bpG9;h;Bd+G5Mb%QNzFEE_?9k5uRzhgc)(gsS- za9F}JnRO>d1J{yaUJhL*)GtANFJ@}D;dEV(KU z%kX$cHGVG%#SeT>GbQ8sETZkmYV4-G3iGnYmV`*MR2Lx2Z<}_aYQ)v%)(9*|E^=66 zDws7i*LOtZ!n{=8Cgb@mqTfsA`@Xp4+`We+M3Uu3A+n6sJGTP5g38tN4}m4-lFUZj t6)KXLTE1${+zdFU170LR>Np~0MR*+Ecb(2bJ^Nq{iP5f}rCjPX{}0iu2n_%L literal 0 HcmV?d00001 diff --git a/logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/trial_1/flags_0.json b/logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/trial_1/flags_0.json new file mode 100644 index 00000000..668bd6d3 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/trial_1/flags_0.json @@ -0,0 +1,62 @@ +{ + "logtostderr": false, + "alsologtostderr": false, + "log_dir": "", + "v": 0, + "verbosity": 0, + "logger_levels": {}, + "stderrthreshold": "fatal", + "showprefixforinfo": true, + "run_with_pdb": false, + "pdb_post_mortem": false, + "pdb": false, + "run_with_profiling": false, + "profile_file": null, + "use_cprofile_for_profiling": true, + "only_check_args": false, + "op_conversion_fallback_to_while_loop": true, + "delta_threshold": 0.5, + "tt_check_filter": false, + "tt_single_core_summaries": false, + "runtime_oom_exit": true, + "hbm_oom_exit": true, + "test_srcdir": "", + "test_tmpdir": "/tmp/absl_testing", + "test_random_seed": 301, + "test_randomize_ordering_seed": "", + "xml_output_file": "", + "experimental_orbax_use_distributed_process_id": false, + "experimental_orbax_use_distributed_barrier": false, + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "workload": "librispeech_conformer", + "tuning_ruleset": "self", + "tuning_search_space": null, + "num_tuning_trials": 1, + "data_dir": "/data/librispeech", + "imagenet_v2_data_dir": null, + "librispeech_tokenizer_vocab_path": "/data/librispeech/spm_model.vocab", + "framework": "pytorch", + "torch_compile": true, + "experiment_dir": "/experiment_runs", + "experiment_name": "submissions_a100/ademamix_golden/study_2", + "save_checkpoints": false, + "save_intermediate_checkpoints": true, + "resume_last_run": null, + "append_timestamp": false, + "use_wandb": false, + "profile": false, + "max_global_steps": null, + "overwrite": true, + "hparam_start_index": null, + "hparam_end_index": null, + "rng_seed": -582177137, + "set_pytorch_max_split_size": false, + "pytorch_eval_num_workers": 0, + "capture_jax_trace": false, + "skip_evals": false, + "?": false, + "help": false, + "helpshort": false, + "helpfull": false, + "helpxml": false +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/trial_1/measurements.csv b/logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/trial_1/measurements.csv new file mode 100644 index 00000000..92ffa60f --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/trial_1/measurements.csv @@ -0,0 +1,320 @@ +global_step,grad_norm,loss,train/ctc_loss,train/wer,validation/ctc_loss,validation/wer,validation/num_examples,test/ctc_loss,test/wer,test/num_examples,score,total_duration,accumulated_submission_time,accumulated_eval_time,accumulated_logging_time,preemption_count +0,0.4999999701976776,33.37062072753906,,,,,,,,,,,,,, +1,,,32.10654239086722,1.7203084470202628,30.97378880349608,1.854255781393328,5348.0,31.06509452222724,1.870249629313672,2472.0,14.052122116088867,78.18270134925842,14.052122116088867,63.21826100349426,0.0,0.0 +1,0.4999999105930328,32.63825225830078,,,,,,,,,,,,,, +2,0.4999999701976776,33.139434814453125,,,,,,,,,,,,,, +3,0.4999999701976776,33.23611068725586,,,,,,,,,,,,,, +4,0.4999999403953552,32.741722106933594,,,,,,,,,,,,,, +5,0.4999999403953552,33.05787658691406,,,,,,,,,,,,,, +6,0.4999999403953552,33.02008056640625,,,,,,,,,,,,,, +7,0.5,31.94777107238769,,,,,,,,,,,,,, +8,0.5,32.48036193847656,,,,,,,,,,,,,, +9,0.4999999701976776,32.03805160522461,,,,,,,,,,,,,, +10,0.4999999701976776,31.97547912597656,,,,,,,,,,,,,, +11,0.5,32.025970458984375,,,,,,,,,,,,,, +12,0.4999999701976776,32.1755485534668,,,,,,,,,,,,,, +13,0.5,31.57413673400879,,,,,,,,,,,,,, +14,0.5,31.33147621154785,,,,,,,,,,,,,, +15,0.5,30.40168571472168,,,,,,,,,,,,,, +16,0.5,31.012741088867188,,,,,,,,,,,,,, +17,0.5,30.030105590820312,,,,,,,,,,,,,, +18,0.4999999701976776,29.67608642578125,,,,,,,,,,,,,, +19,0.5,29.099021911621094,,,,,,,,,,,,,, +20,0.4999999403953552,28.13345718383789,,,,,,,,,,,,,, +21,0.5,27.84108543395996,,,,,,,,,,,,,, +22,0.5,28.0482120513916,,,,,,,,,,,,,, +23,0.5,26.3504581451416,,,,,,,,,,,,,, +24,0.4999999701976776,25.966062545776367,,,,,,,,,,,,,, +25,0.4999999701976776,25.312271118164062,,,,,,,,,,,,,, +26,0.4999999701976776,24.32256507873535,,,,,,,,,,,,,, +27,0.4999999701976776,23.31526756286621,,,,,,,,,,,,,, +28,0.5,21.90838623046875,,,,,,,,,,,,,, +29,0.5,22.107908248901367,,,,,,,,,,,,,, +30,0.5000000596046448,21.187721252441406,,,,,,,,,,,,,, +31,0.5,20.739717483520508,,,,,,,,,,,,,, +32,0.5,20.184789657592773,,,,,,,,,,,,,, +33,0.4999999701976776,19.569549560546875,,,,,,,,,,,,,, +34,0.5,19.715600967407227,,,,,,,,,,,,,, +35,0.5,19.31510353088379,,,,,,,,,,,,,, +36,0.5,18.21173667907715,,,,,,,,,,,,,, +37,0.4999999701976776,17.653121948242188,,,,,,,,,,,,,, +38,0.4999999701976776,17.000118255615234,,,,,,,,,,,,,, +39,0.5,17.025297164916992,,,,,,,,,,,,,, +40,0.4999999403953552,16.571552276611328,,,,,,,,,,,,,, +41,0.4999999403953552,15.931239128112791,,,,,,,,,,,,,, +42,0.4999999701976776,15.423179626464844,,,,,,,,,,,,,, +43,0.4999999403953552,15.051555633544922,,,,,,,,,,,,,, +44,0.4999999701976776,15.193470001220703,,,,,,,,,,,,,, +45,0.4999999403953552,14.766130447387695,,,,,,,,,,,,,, +46,0.4999999105930328,13.583130836486816,,,,,,,,,,,,,, +47,0.4999999701976776,13.504735946655272,,,,,,,,,,,,,, +48,0.4999999105930328,14.267671585083008,,,,,,,,,,,,,, +49,0.4999999403953552,14.865878105163574,,,,,,,,,,,,,, +50,0.5,13.926901817321776,,,,,,,,,,,,,, +51,0.4999999701976776,12.410497665405272,,,,,,,,,,,,,, +52,0.5,12.630412101745604,,,,,,,,,,,,,, +53,0.4999999105930328,12.602208137512209,,,,,,,,,,,,,, +54,0.4999999403953552,13.21047019958496,,,,,,,,,,,,,, +55,0.4999999403953552,12.48936939239502,,,,,,,,,,,,,, +56,0.4999999403953552,12.59957790374756,,,,,,,,,,,,,, +57,0.4999999105930328,11.966904640197754,,,,,,,,,,,,,, +58,0.4999999701976776,12.116439819335938,,,,,,,,,,,,,, +59,0.4999999403953552,11.612298965454102,,,,,,,,,,,,,, +60,0.4999999701976776,11.619709014892578,,,,,,,,,,,,,, +61,0.4999999403953552,11.17806625366211,,,,,,,,,,,,,, +62,0.4999999105930328,11.417095184326172,,,,,,,,,,,,,, +63,0.4999999403953552,11.356446266174316,,,,,,,,,,,,,, +64,0.4999999105930328,10.410043716430664,,,,,,,,,,,,,, +65,0.4999999403953552,11.162188529968262,,,,,,,,,,,,,, +66,0.4999999105930328,11.281346321105955,,,,,,,,,,,,,, +67,0.4999999403953552,11.060033798217772,,,,,,,,,,,,,, +68,0.4999999701976776,10.636923789978027,,,,,,,,,,,,,, +69,0.4999999105930328,11.148841857910156,,,,,,,,,,,,,, +70,0.4999998211860657,10.66979694366455,,,,,,,,,,,,,, +71,0.4999999105930328,11.124584197998049,,,,,,,,,,,,,, +72,0.4999998807907104,10.430634498596191,,,,,,,,,,,,,, +73,0.4999999403953552,10.626057624816896,,,,,,,,,,,,,, +74,0.4999999105930328,10.969554901123049,,,,,,,,,,,,,, +75,0.4999999701976776,10.628463745117188,,,,,,,,,,,,,, +76,0.4999999701976776,9.97251796722412,,,,,,,,,,,,,, +77,0.4999999701976776,10.08691692352295,,,,,,,,,,,,,, +78,0.4999999403953552,9.981465339660645,,,,,,,,,,,,,, +79,0.4999998807907104,9.258590698242188,,,,,,,,,,,,,, +80,0.5,9.080510139465332,,,,,,,,,,,,,, +81,0.4999999403953552,8.535042762756348,,,,,,,,,,,,,, +82,0.4999999701976776,7.441348075866699,,,,,,,,,,,,,, +83,0.4999999701976776,7.221118450164795,,,,,,,,,,,,,, +84,0.4999999701976776,7.407680034637451,,,,,,,,,,,,,, +85,0.4999999701976776,7.364335060119629,,,,,,,,,,,,,, +86,0.4999999105930328,7.1868181228637695,,,,,,,,,,,,,, +87,0.4999999403953552,6.882572650909424,,,,,,,,,,,,,, +88,0.4999999105930328,6.592681407928467,,,,,,,,,,,,,, +89,0.4999999105930328,6.593204498291016,,,,,,,,,,,,,, +90,0.4999999105930328,6.7307658195495605,,,,,,,,,,,,,, +91,0.4999999403953552,6.773481369018555,,,,,,,,,,,,,, +92,0.4999999701976776,6.601720809936523,,,,,,,,,,,,,, +93,0.4999999701976776,6.442417621612549,,,,,,,,,,,,,, +94,0.4999999105930328,6.470370769500732,,,,,,,,,,,,,, +95,0.4999999105930328,6.475524425506592,,,,,,,,,,,,,, +96,0.4999999701976776,6.436556816101074,,,,,,,,,,,,,, +97,0.4999997615814209,6.35900354385376,,,,,,,,,,,,,, +98,0.4999999701976776,6.374427795410156,,,,,,,,,,,,,, +99,0.4999999701976776,6.422395706176758,,,,,,,,,,,,,, +100,0.4999999701976776,6.385075092315674,,,,,,,,,,,,,, +500,0.499999850988388,5.769547462463379,,,,,,,,,,,,,, +1000,0.4999995231628418,3.565702438354492,,,,,,,,,,,,,, +1409,,,4.387652035647058,0.7925199969611131,4.561761838833634,0.7672765895814223,5348.0,4.362626217820504,0.756240732841793,2472.0,1760.6268198490145,1875.831004858017,1760.6268198490145,111.74717354774477,0.0366904735565185,0.0 +1500,0.4999993443489074,2.76863980293274,,,,,,,,,,,,,, +2000,0.4999993145465851,2.2968451976776123,,,,,,,,,,,,,, +2500,0.4999993145465851,2.237355709075928,,,,,,,,,,,,,, +3000,0.4999992251396179,1.9697719812393188,,,,,,,,,,,,,, +3026,,,1.3805324669399563,0.4144064945354301,1.5873540748380048,0.4231352290831844,5348.0,1.2777354587392735,0.3753986147502691,2472.0,3506.216897964477,3674.096529006958,3506.216897964477,161.74285888671875,0.0768518447875976,0.0 +3500,0.4999991655349731,1.87246322631836,,,,,,,,,,,,,, +4000,0.4999993145465851,1.8377220630645752,,,,,,,,,,,,,, +4500,0.4999992549419403,1.7452819347381592,,,,,,,,,,,,,, +5000,0.4999992251396179,1.7629717588424685,,,,,,,,,,,,,, +5304,,,0.5142548732515267,0.1753546272479623,0.724086608948036,0.2171003717472119,5348.0,0.4732933878798632,0.1583287632279162,2472.0,5251.343508243561,5468.964254617691,5251.343508243561,208.47549104690552,0.1181724071502685,0.0 +5500,0.4999993443489074,1.7153592109680176,,,,,,,,,,,,,, +6000,0.499999076128006,1.5880098342895508,,,,,,,,,,,,,, +6500,0.4999991655349731,1.5242141485214231,,,,,,,,,,,,,, +7000,0.4999993443489074,1.5980032682418823,,,,,,,,,,,,,, +7500,0.4999991059303283,1.5267540216445925,,,,,,,,,,,,,, +7782,,,0.4007701530663972,0.1409446596989331,0.6215434749849307,0.1870612658716748,5348.0,0.3864171428398606,0.1295472548900128,2472.0,6996.924051761627,7262.596936225891,6996.924051761627,253.65583658218384,0.1604683399200439,0.0 +8000,0.4999990463256836,1.5462758541107178,,,,,,,,,,,,,, +8500,0.4999991059303283,1.484225153923035,,,,,,,,,,,,,, +9000,0.4999991357326507,1.530664086341858,,,,,,,,,,,,,, +9500,0.4999993741512298,1.5108195543289185,,,,,,,,,,,,,, +10000,0.4999992251396179,1.508371114730835,,,,,,,,,,,,,, +10121,,,0.3454691138252321,0.1227004851365871,0.5673220839298272,0.1700477960701009,5348.0,0.3434555253564746,0.1161213007535596,2472.0,8744.24173116684,9059.876303434372,8744.24173116684,299.8750274181366,0.2235958576202392,0.0 +10500,0.4999992847442627,1.5165212154388428,,,,,,,,,,,,,, +11000,0.4999991655349731,1.4546624422073364,,,,,,,,,,,,,, +11500,0.4999991655349731,1.3893704414367676,,,,,,,,,,,,,, +12000,0.4999992251396179,1.4320924282073977,,,,,,,,,,,,,, +12489,,,0.3102659863434054,0.1111038756661131,0.5330505187142104,0.1594361029305267,5348.0,0.3173984007032712,0.1076716836268356,2472.0,10489.384288549423,10856.218958854675,10489.384288549423,347.81264901161194,0.540952205657959,0.0 +12500,0.499999076128006,1.3645166158676147,,,,,,,,,,,,,, +13000,0.4999991059303283,1.450552225112915,,,,,,,,,,,,,, +13500,0.4999990165233612,1.3179093599319458,,,,,,,,,,,,,, +14000,0.4999991655349731,1.4198611974716189,,,,,,,,,,,,,, +14500,0.4999990463256836,1.343605875968933,,,,,,,,,,,,,, +15000,0.4999990463256836,1.4101194143295288,,,,,,,,,,,,,, +15002,,,0.2772388136150266,0.1004840512703632,0.502230916512834,0.1494810022691063,5348.0,0.297861303551842,0.0991814433408486,2472.0,12234.653105258942,12650.925751447678,12234.653105258942,393.4130597114563,0.6633203029632568,0.0 +15500,0.493755042552948,1.3099535703659058,,,,,,,,,,,,,, +16000,0.4637251794338226,1.3456053733825684,,,,,,,,,,,,,, +16500,0.499999076128006,1.3552284240722656,,,,,,,,,,,,,, +17000,0.4999993145465851,1.3528531789779663,,,,,,,,,,,,,, +17340,,,0.2548133972595564,0.092962806194988,0.475153822819344,0.1422488292376768,5348.0,0.2780792309181237,0.0940223021144354,2472.0,13979.592563390732,14446.687247037888,13979.592563390732,440.97758173942566,1.0245919227600098,0.0 +17500,0.4469206035137176,1.3230818510055542,,,,,,,,,,,,,, +18000,0.4999991357326507,1.3937314748764038,,,,,,,,,,,,,, +18500,0.499999076128006,1.277355670928955,,,,,,,,,,,,,, +19000,0.470878005027771,1.258722186088562,,,,,,,,,,,,,, +19500,0.4999991357326507,1.3918864727020264,,,,,,,,,,,,,, +19706,,,0.2362978348332719,0.0858377017332508,0.46606361921087,0.1356925602278762,5348.0,0.2691817314826763,0.0881928787601811,2472.0,15724.52820301056,16238.72852897644,15724.52820301056,484.9104630947113,1.1978044509887695,0.0 +20000,0.4392909407615661,1.3555222749710083,,,,,,,,,,,,,, +20500,0.499999076128006,1.2952470779418943,,,,,,,,,,,,,, +21000,0.499999076128006,1.3193025588989258,,,,,,,,,,,,,, +21500,0.4850276112556457,1.3551713228225708,,,,,,,,,,,,,, +22000,0.4778873026371002,1.2153838872909546,,,,,,,,,,,,,, +22219,,,0.221420801953187,0.0829181996765756,0.4489882953774864,0.1326896152174962,5348.0,0.2557915026775921,0.0842930554709239,2472.0,17469.457185268402,18039.508671998978,17469.457185268402,536.7038400173187,1.4543979167938232,0.0 +22500,0.4999994337558746,1.2746024131774902,,,,,,,,,,,,,, +23000,0.4679112732410431,1.2840261459350586,,,,,,,,,,,,,, +23500,0.4999989867210388,1.166265845298767,,,,,,,,,,,,,, +24000,0.4999993443489074,1.2676396369934082,,,,,,,,,,,,,, +24500,0.4999991953372955,1.269620060920715,,,,,,,,,,,,,, +24558,,,0.206900756080581,0.0762217953309673,0.4338568253811282,0.1275237773379037,5348.0,0.2476686499935479,0.0815103690614019,2472.0,19214.5674712658,19834.014476537704,19214.5674712658,583.007815361023,1.736563205718994,0.0 +25000,0.499999076128006,1.294004201889038,,,,,,,,,,,,,, +25500,0.4999989867210388,1.235024094581604,,,,,,,,,,,,,, +26000,0.4999991655349731,1.2090359926223757,,,,,,,,,,,,,, +26500,0.4999992549419403,1.276907205581665,,,,,,,,,,,,,, +26938,,,0.1976364138681518,0.0744093163589793,0.4246268185214487,0.1247236035340124,5348.0,0.2418726107329505,0.0804947900798245,2472.0,20960.067337989807,21626.68429684639,20960.067337989807,627.1416909694672,1.819189548492432,0.0 +27000,0.4999996125698089,1.177448868751526,,,,,,,,,,,,,, +27500,0.4999992549419403,1.2137395143508911,,,,,,,,,,,,,, +28000,0.4999991059303283,1.2016099691390991,,,,,,,,,,,,,, +28500,0.4999991357326507,1.111467957496643,,,,,,,,,,,,,, +29000,0.4999990463256836,1.1350462436676023,,,,,,,,,,,,,, +29454,,,0.1849809459075839,0.0696502024115738,0.4099803186454566,0.1196446676000579,5348.0,0.2330298779356732,0.074807547782991,2472.0,22705.55812382698,23424.6008489132,22705.55812382698,675.4134092330933,2.2177481651306152,0.0 +29500,0.4999991953372955,1.206831693649292,,,,,,,,,,,,,, +30000,0.4254769682884216,1.2787669897079468,,,,,,,,,,,,,, +30500,0.424522191286087,1.2213572263717651,,,,,,,,,,,,,, +31000,0.4999992251396179,1.1581604480743408,,,,,,,,,,,,,, +31500,0.4777238667011261,1.1857991218566897,,,,,,,,,,,,,, +31784,,,0.1756894884184237,0.0657213557776837,0.4060511360068816,0.117076232317868,5348.0,0.2263144245193238,0.0732841793106249,2472.0,24452.78587174416,25220.40988755226,24452.78587174416,720.7431962490082,2.602026462554932,0.0 +32000,0.4999989867210388,1.1973164081573486,,,,,,,,,,,,,, +32500,0.409768134355545,1.181714415550232,,,,,,,,,,,,,, +33000,0.4999990463256836,1.1470028162002563,,,,,,,,,,,,,, +33500,0.4999995231628418,1.2114722728729248,,,,,,,,,,,,,, +34000,0.499999076128006,1.1553425788879397,,,,,,,,,,,,,, +34184,,,0.1658531643607152,0.0623297409348918,0.3972254793927064,0.1144788297204654,5348.0,0.2211009833295696,0.0696484065565779,2472.0,26198.30634236336,27015.766971111298,26198.30634236336,767.1031429767609,2.691215991973877,0.0 +34500,0.4999991655349731,1.1476751565933228,,,,,,,,,,,,,, +35000,0.4999993443489074,1.1275880336761477,,,,,,,,,,,,,, +35500,0.4822714626789093,1.0950089693069458,,,,,,,,,,,,,, +36000,0.4999992847442627,1.1298199892044067,,,,,,,,,,,,,, +36500,0.491209477186203,1.123528242111206,,,,,,,,,,,,,, +36676,,,0.1571575696981423,0.0598932048318301,0.3931514164374874,0.1118910828948003,5348.0,0.2171341195399703,0.0693437328621046,2472.0,27943.172302246094,28813.034168481827,27943.172302246094,815.4270529747009,3.0661959648132324,0.0 +37000,0.4999991357326507,1.0901851654052734,,,,,,,,,,,,,, +37500,0.3879214525222778,1.177807092666626,,,,,,,,,,,,,, +38000,0.4999989867210388,1.119940161705017,,,,,,,,,,,,,, +38500,0.4999991953372955,1.115863800048828,,,,,,,,,,,,,, +39000,0.4999989569187164,1.0938454866409302,,,,,,,,,,,,,, +39002,,,0.1511255163255616,0.0578202498399157,0.3916050698447232,0.1102109786124656,5348.0,0.2171851308955416,0.0677391180712124,2472.0,29688.63876318932,30607.621863126755,29688.63876318932,861.4726457595825,3.345543146133423,0.0 +39500,0.4881621897220611,1.061312198638916,,,,,,,,,,,,,, +40000,0.4327720105648041,1.1775567531585691,,,,,,,,,,,,,, +40500,0.4999989569187164,1.1031936407089231,,,,,,,,,,,,,, +41000,0.4999991953372955,1.125430941581726,,,,,,,,,,,,,, +41409,,,0.1440294721672113,0.0549007477832405,0.3844594313529736,0.1080673972867281,5348.0,0.2121576139992902,0.0670282127841082,2472.0,31433.99880838394,32405.74048280716,31433.99880838394,910.729204416275,3.4501547813415527,0.0 +41500,0.4999993443489074,1.1819593906402588,,,,,,,,,,,,,, +42000,0.4830639362335205,1.1498509645462036,,,,,,,,,,,,,, +42500,0.499999076128006,1.126837968826294,,,,,,,,,,,,,, +43000,0.4796006083488464,1.0872279405593872,,,,,,,,,,,,,, +43500,0.499999076128006,1.1292104721069336,,,,,,,,,,,,,, +43899,,,0.1363457895334926,0.0523394002539641,0.3755043708873317,0.1064935064935064,5348.0,0.2063891848869282,0.0658095180062153,2472.0,33179.23314833641,34202.98544931412,33179.23314833641,958.8653280735016,3.728386402130127,0.0 +44000,0.4953942894935608,1.073352932929993,,,,,,,,,,,,,, +44500,0.4999990165233612,1.102705001831055,,,,,,,,,,,,,, +45000,0.4999992549419403,1.0850733518600464,,,,,,,,,,,,,, +45500,0.4585126340389251,1.051321268081665,,,,,,,,,,,,,, +46000,0.499998927116394,0.9928967952728271,,,,,,,,,,,,,, +46223,,,0.1278055994656996,0.0496261083797306,0.3676198108957454,0.1031332979288369,5348.0,0.1997780602780824,0.0622143684114313,2472.0,34924.75255584717,35996.56893110275,34924.75255584717,1003.7615249156952,4.018621206283569,0.0 +46500,0.4999996125698089,1.017992377281189,,,,,,,,,,,,,, +47000,0.4545561373233795,1.0704352855682373,,,,,,,,,,,,,, +47500,0.4999994337558746,1.1044715642929075,,,,,,,,,,,,,, +48000,0.4999993443489074,1.0236016511917114,,,,,,,,,,,,,, +48500,0.4999993443489074,1.0751190185546875,,,,,,,,,,,,,, +48661,,,0.1207196938334105,0.046435277135632,0.3633735201457956,0.1023415246463573,5348.0,0.1967958996346538,0.0624174842077468,2472.0,36669.94425535202,37794.34977936745,36669.94425535202,1052.7004284858704,4.158437013626099,0.0 +49000,0.4999992847442627,1.005373477935791,,,,,,,,,,,,,, +49500,0.4999990165233612,1.1196348667144775,,,,,,,,,,,,,, +50000,0.4544858038425445,1.0399775505065918,,,,,,,,,,,,,, +50500,0.4999991059303283,1.0215626955032349,,,,,,,,,,,,,, +51000,0.4999990463256836,1.0326577425003052,,,,,,,,,,,,,, +51140,,,0.1124454156542221,0.0434343763227297,0.3598224439845163,0.0999855163423936,5348.0,0.1932157543430221,0.059980094651961,2472.0,38414.99412441254,39588.95394158363,38414.99412441254,1098.2790446281433,4.471155881881714,0.0 +51500,0.499999076128006,1.0938210487365725,,,,,,,,,,,,,, +52000,0.4855721294879913,1.0175474882125854,,,,,,,,,,,,,, +52500,0.4999994039535522,0.9778187870979308,,,,,,,,,,,,,, +53000,0.4999991655349731,1.0625840425491333,,,,,,,,,,,,,, +53453,,,0.1056231116995211,0.0410141199709135,0.3530767163718731,0.0979481485057693,5348.0,0.1893275246588489,0.0590457619889098,2472.0,40161.32847809792,41384.607890844345,40161.32847809792,1144.4815368652344,4.797126770019531,0.0 +53500,0.4999991357326507,1.0316861867904663,,,,,,,,,,,,,, +54000,0.4999991357326507,1.0643409490585327,,,,,,,,,,,,,, +54500,0.4999992251396179,1.0276998281478882,,,,,,,,,,,,,, +55000,0.4999992549419403,0.9919557571411132,,,,,,,,,,,,,, +55500,0.3879493772983551,0.9818668961524964,,,,,,,,,,,,,, +55898,,,0.0991121581838182,0.0379209672342873,0.3445919098948915,0.0950803842997151,5348.0,0.1878857229700303,0.0572380314017021,2472.0,41906.71695399284,43179.3950483799,41906.71695399284,1189.9835596084597,4.889002799987793,0.0 +56000,0.499999076128006,0.9745871424674988,,,,,,,,,,,,,, +56500,0.4999993443489074,1.0378713607788086,,,,,,,,,,,,,, +57000,0.499999076128006,1.0187232494354248,,,,,,,,,,,,,, +57500,0.4999991953372955,0.9819082617759703,,,,,,,,,,,,,, +58000,0.4999992251396179,1.0360758304595947,,,,,,,,,,,,,, +58362,,,0.0930589300771203,0.0365426149621767,0.3463119558955571,0.0938541012890455,5348.0,0.1841833545712626,0.0560396482034407,2472.0,43651.70753669739,44974.58230113983,43651.70753669739,1236.2817873954773,5.1664628982543945,0.0 +58500,0.4999994039535522,1.0322787761688232,,,,,,,,,,,,,, +59000,0.4999991357326507,0.9901466369628906,,,,,,,,,,,,,, +59500,0.4999991357326507,1.014813780784607,,,,,,,,,,,,,, +60000,0.4999993741512298,1.0225430727005005,,,,,,,,,,,,,, +60500,0.4999991953372955,0.9610987901687622,,,,,,,,,,,,,, +60677,,,0.0892244418318964,0.034686723320201,0.3447293340381379,0.0924926374740501,5348.0,0.1844395076900122,0.0556943513497044,2472.0,45398.96501159668,46769.41661047936,45398.96501159668,1280.783971786499,5.473428726196289,0.0 +61000,0.4999992847442627,1.0309436321258545,,,,,,,,,,,,,, +61500,0.499999076128006,0.9746392369270324,,,,,,,,,,,,,, +62000,0.4999992251396179,1.0013296604156494,,,,,,,,,,,,,, +62500,0.4837281405925751,1.0057477951049805,,,,,,,,,,,,,, +63000,0.4999993443489074,0.9603309631347656,,,,,,,,,,,,,, +63162,,,0.0844593807874086,0.0330967342819001,0.3397519458650417,0.0912566986916429,5348.0,0.1807859705666494,0.0537647512847074,2472.0,47144.11619091034,48564.43762612343,47144.11619091034,1326.9350910186768,5.58700704574585,0.0 +63500,0.4771880507469177,0.9989185333251952,,,,,,,,,,,,,, +64000,0.4999990463256836,1.0042109489440918,,,,,,,,,,,,,, +64500,0.4999991059303283,0.951124668121338,,,,,,,,,,,,,, +65000,0.4999990463256836,0.964393675327301,,,,,,,,,,,,,, +65500,0.4999991655349731,0.9136340618133544,,,,,,,,,,,,,, +65602,,,0.0803655680552786,0.0315338781623416,0.3392857633394866,0.0898952348766475,5348.0,0.1796280329577714,0.0533585196920764,2472.0,48890.07870817184,50361.868340969086,48890.07870817184,1374.4993312358856,5.889404773712158,0.0 +66000,0.4999992251396179,0.9275556802749634,,,,,,,,,,,,,, +66500,0.4999991953372955,0.9598451852798462,,,,,,,,,,,,,, +67000,0.4999992847442627,0.9166220426559448,,,,,,,,,,,,,, +67500,0.4999994039535522,0.9201321601867676,,,,,,,,,,,,,, +67911,,,0.0776614528592333,0.0302640575652004,0.3344761390678371,0.089209675083281,5348.0,0.1773310853724433,0.0523835638697621,2472.0,50637.22282648087,52158.77340841293,50637.22282648087,1421.1971504688263,6.198186874389648,0.0 +68000,0.4999996125698089,0.9116886854171752,,,,,,,,,,,,,, +68500,0.4999992549419403,0.8540123701095581,,,,,,,,,,,,,, +69000,0.4999993741512298,0.9951090216636658,,,,,,,,,,,,,, +69500,0.4999990463256836,0.9425287246704102,,,,,,,,,,,,,, +70000,0.4999995231628418,1.0565874576568604,,,,,,,,,,,,,, +70423,,,0.0758885375197278,0.0294663497541757,0.335301566895218,0.0888813788442041,5348.0,0.1769064359999032,0.0522616943919728,2472.0,52382.57315540314,53954.31312060356,52382.57315540314,1467.5598187446594,6.286574602127075,0.0 +70500,0.4999994039535522,0.9557844996452332,,,,,,,,,,,,,, +71000,0.4999996125698089,0.9248284101486206,,,,,,,,,,,,,, +71500,0.4999990463256836,0.9631268382072448,,,,,,,,,,,,,, +72000,0.4999994337558746,0.941565990447998,,,,,,,,,,,,,, +72500,0.4999991953372955,0.9103706479072572,,,,,,,,,,,,,, +72847,,,0.0747346069187836,0.029086488891783,0.3327777993614376,0.0880702940182494,5348.0,0.1756454045422285,0.0519570206974996,2472.0,54128.10344576836,55750.47462272644,54128.10344576836,1514.0154871940613,6.655657052993774,0.0 +73000,0.4999990463256836,0.8772071599960327,,,,,,,,,,,,,, +73500,0.4999993741512298,1.0530591011047363,,,,,,,,,,,,,, +74000,0.4593827426433563,0.8975034952163696,,,,,,,,,,,,,, +74500,0.4999993145465851,0.9453960061073304,,,,,,,,,,,,,, +75000,0.4999991953372955,0.9670403003692628,,,,,,,,,,,,,, +75145,,,0.0745783904498464,0.0289508242980713,0.3340026475302014,0.0883116883116883,5348.0,0.1761567404550293,0.0522210712327097,2472.0,55875.95197558403,57548.5589632988,55875.95197558403,1561.195592880249,6.92978048324585,0.0 +75500,0.4999990165233612,0.9355252385139464,,,,,,,,,,,,,, +76000,0.4999992549419403,0.9356448650360109,,,,,,,,,,,,,, +76500,0.4999992847442627,0.9645879864692688,,,,,,,,,,,,,, +77000,0.4999995827674866,0.95625638961792,,,,,,,,,,,,,, +77500,0.4999997019767761,0.9298882484436036,,,,,,,,,,,,,, +77691,,,0.0744802394846587,0.0288748521255928,0.3337463660965441,0.0884179018008014,5348.0,0.1759873620878766,0.0519976438567627,2472.0,57621.09947395325,59344.91904091835,57621.09947395325,1608.6839032173157,7.007215261459351,0.0 +78000,0.4999992847442627,0.9079465866088868,,,,,,,,,,,,,, +78500,0.4999993443489074,0.9622384905815125,,,,,,,,,,,,,, +79000,0.4999993443489074,0.9754884839057922,,,,,,,,,,,,,, +79500,0.4999993443489074,0.86479651927948,,,,,,,,,,,,,, +80000,0.4999996423721313,0.9389192461967468,,,,,,,,,,,,,, +80087,,,0.0740967642577247,0.0287391875318811,0.3334598430122312,0.0879061458987109,5348.0,0.1755548871096522,0.0512461154103954,2472.0,59368.39318227768,61143.36517834664,59368.39318227768,1655.7432885169983,7.346552133560181,0.0 +80500,0.4999992847442627,0.9409658312797546,,,,,,,,,,,,,, +81000,0.4999997317790985,0.9207749366760254,,,,,,,,,,,,,, +81500,0.499999076128006,0.9397523999214172,,,,,,,,,,,,,, +82000,0.4999993145465851,0.9980356693267822,,,,,,,,,,,,,, +82379,,,0.0736546360733826,0.0287337609481327,0.3340219747165084,0.0877999324095978,5348.0,0.1755761964481579,0.051713281741921,2472.0,61113.596465587616,62936.45216083527,61113.596465587616,1700.4697089195251,7.771591901779175,0.0 +82500,0.4999994039535522,0.9851810336112976,,,,,,,,,,,,,, +83000,0.4999990463256836,0.912049114704132,,,,,,,,,,,,,, +83500,0.4999994933605194,0.9765385985374452,,,,,,,,,,,,,, +84000,0.4999995231628418,0.9469825625419616,,,,,,,,,,,,,, +84500,0.4999993741512298,0.95150887966156,,,,,,,,,,,,,, +84938,,,0.073440406905519,0.0287337609481327,0.3364899435842375,0.0883309998551634,5348.0,0.1764033919527066,0.051936709117868,2472.0,62859.03667855263,64733.03278779984,62859.03667855263,1747.5974349975586,7.8443779945373535,0.0 +85000,0.4999994039535522,0.9607521295547484,,,,,,,,,,,,,, +85500,0.4999991655349731,1.0015202760696411,,,,,,,,,,,,,, +86000,0.4999993145465851,0.8951892256736755,,,,,,,,,,,,,, +86500,0.4999994039535522,0.9680371284484864,,,,,,,,,,,,,, +87000,0.4999992847442627,0.8893512487411499,,,,,,,,,,,,,, +87318,,,0.0737856844434456,0.0288585723743474,0.3393105354945248,0.0883406556269009,5348.0,0.1797198861620104,0.0515711006845002,2472.0,64606.470702409744,66530.33806610107,64606.470702409744,1793.6775379180908,8.162017583847046,0.0 +87500,0.4999995529651642,0.9532162547111512,,,,,,,,,,,,,, +88000,0.4999994933605194,0.9181004166603088,,,,,,,,,,,,,, +88500,0.4999992549419403,0.9086982011795044,,,,,,,,,,,,,, +89000,0.4999992251396179,0.9649962186813354,,,,,,,,,,,,,, +89500,0.4999995529651642,0.9639212489128112,,,,,,,,,,,,,, +89611,,,,,,,,,,,66353.51962351799,,,,,0.0 diff --git a/logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/trial_1/meta_data_0.json b/logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/trial_1/meta_data_0.json new file mode 100644 index 00000000..d0fc3dde --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/librispeech_conformer_pytorch/trial_1/meta_data_0.json @@ -0,0 +1,71 @@ +{ + "workload.attention_temperature": 1.0, + "workload.eval_batch_size": 256, + "workload.eval_num_workers": 0, + "workload.eval_period_time_sec": 1747, + "workload.max_allowed_runtime_sec": 43680, + "workload.num_eval_train_examples": 5376, + "workload.num_test_examples": 2472, + "workload.num_train_examples": 263840, + "workload.num_validation_examples": 5348, + "workload.requires_sync_before_eval": true, + "workload.step_hint": 76000, + "workload.target_metric_name": "wer", + "workload.test_target_value": 0.052981, + "workload.use_gelu": false, + "workload.use_post_layer_norm": true, + "workload.use_specaug": true, + "workload.validation_target_value": 0.085884, + "cpu.util.avg_percent_since_last": 15.8, + "cpu.freq.current": 2200.1639999999993, + "mem.total": 359053524992, + "mem.available": 349484081152, + "mem.used": 6413631488, + "mem.percent_used": 2.7, + "mem.read_bytes_since_boot": 48901126804480, + "mem.write_bytes_since_boot": 138422502400, + "net.bytes_sent_since_boot": 31216954, + "net.bytes_recv_since_boot": 31219916, + "gpu.count": 4, + "gpu.0.compute.util": 0.08, + "gpu.0.mem.util": 0.0333251953125, + "gpu.0.mem.total": 40960.0, + "gpu.0.mem.used": 1365.0, + "gpu.0.mem.free": 38962.0, + "gpu.0.temp.current": 37.0, + "gpu.1.compute.util": 0.05, + "gpu.1.mem.util": 0.0333251953125, + "gpu.1.mem.total": 40960.0, + "gpu.1.mem.used": 1365.0, + "gpu.1.mem.free": 38962.0, + "gpu.1.temp.current": 42.0, + "gpu.2.compute.util": 0.0, + "gpu.2.mem.util": 0.0333251953125, + "gpu.2.mem.total": 40960.0, + "gpu.2.mem.used": 1365.0, + "gpu.2.mem.free": 38962.0, + "gpu.2.temp.current": 37.0, + "gpu.3.compute.util": 0.0, + "gpu.3.mem.util": 0.0333251953125, + "gpu.3.mem.total": 40960.0, + "gpu.3.mem.used": 1365.0, + "gpu.3.mem.free": 38962.0, + "gpu.3.temp.current": 41.0, + "gpu.avg.compute.util": 0.0325, + "gpu.avg.mem.util": 0.0333251953125, + "gpu.avg.mem.total": 40960.0, + "gpu.avg.mem.used": 1365.0, + "gpu.avg.mem.free": 38962.0, + "gpu.avg.temp.current": 39.25, + "os_platform": "Linux-6.1.0-44-cloud-amd64-x86_64-with-glibc2.31", + "python_version": "3.11.10", + "python_compiler": "GCC 9.4.0", + "git_branch": "main", + "git_commit_hash": "b21be29be0a1573fb4f78f849aea019cdb520862", + "cpu_model_name": "Intel(R) Xeon(R) CPU @ 2.20GHz", + "cpu_count": 24, + "gpu_model_name": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_driver": "550.90.12", + "rng_seed": -582177137 +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/librispeech_deepspeech_pytorch_09-16-2026-18-08-25.log b/logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/librispeech_deepspeech_pytorch_09-16-2026-18-08-25.log new file mode 100644 index 00000000..2d355ac3 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/librispeech_deepspeech_pytorch_09-16-2026-18-08-25.log @@ -0,0 +1,1035 @@ +torchrun --redirects 1:0,2:0,3:0 --standalone --nnodes=1 --nproc_per_node=4 submission_runner.py --framework=pytorch --workload=librispeech_deepspeech --submission_path=submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py --data_dir=/data/librispeech --experiment_dir=/experiment_runs --experiment_name=submissions_a100/ademamix_golden/study_2 --overwrite=True --save_checkpoints=False --rng_seed=-618110163 --librispeech_tokenizer_vocab_path=/data/librispeech/spm_model.vocab --torch_compile=true --tuning_ruleset=self 2>&1 | tee -a /logs/librispeech_deepspeech_pytorch_09-16-2026-18-08-25.log +W0916 18:08:50.605000 9 site-packages/torch/distributed/run.py:803] +W0916 18:08:50.605000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0916 18:08:50.605000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0916 18:08:50.605000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-16 18:09:04.899136: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 18:09:04.899135: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 18:09:04.899172: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 18:09:04.899178: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789582145.397329 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789582145.397307 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789582145.397329 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789582145.397375 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789582145.503062 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789582145.503076 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789582145.503078 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789582145.503107 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789582146.799692 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789582146.799699 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789582146.799708 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789582146.799709 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789582146.799736 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789582146.799737 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789582146.799738 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789582146.799738 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789582146.799739 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789582146.799740 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789582146.799741 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789582146.799741 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789582146.799742 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789582146.799742 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789582146.799743 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789582146.799744 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789582173.177277 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789582173.177242 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789582173.179833 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789582173.245970 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W916 18:09:41.163120121 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0916 18:09:44.144944 140378530096320 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/librispeech_deepspeech_pytorch. +I0916 18:09:44.144947 139810648466624 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/librispeech_deepspeech_pytorch. +I0916 18:09:44.144957 140298741052608 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/librispeech_deepspeech_pytorch. +I0916 18:09:44.144971 140109961065664 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/librispeech_deepspeech_pytorch. +I0916 18:09:44.367525 140109961065664 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/librispeech_deepspeech_pytorch/trial_1. +I0916 18:09:44.641062 140109961065664 submission_runner.py:242] Initializing dataset. +I0916 18:09:44.641255 140109961065664 input_pipeline.py:19] Loading split = train-clean-100 +I0916 18:09:44.717093 140109961065664 input_pipeline.py:19] Loading split = train-clean-360 +I0916 18:09:44.870586 140109961065664 input_pipeline.py:19] Loading split = train-other-500 +I0916 18:09:45.386482 140109961065664 submission_runner.py:251] Initializing model. +W0916 18:09:58.443450 140109961065664 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0916 18:09:58.445209 140298741052608 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0916 18:09:58.445561 139810648466624 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0916 18:09:58.446426 140378530096320 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +I0916 18:10:02.296339 140109961065664 submission_runner.py:294] Initializing optimizer. +I0916 18:10:02.296917 140109961065664 submission_runner.py:299] Initializing metrics bundle. +I0916 18:10:02.297053 140109961065664 submission_runner.py:321] Initializing checkpoint and logger. +I0916 18:10:02.299300 140109961065664 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_2/librispeech_deepspeech_pytorch/trial_1/meta_data_0.json. +I0916 18:10:02.299417 140298741052608 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 18:10:02.299516 140109961065664 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 18:10:02.299466 140378530096320 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 18:10:02.299482 139810648466624 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 18:10:02.299575 140109961065664 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 18:10:02.299576 140298741052608 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 18:10:02.299625 140378530096320 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 18:10:02.299639 139810648466624 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 18:10:02.759625 140109961065664 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_2/librispeech_deepspeech_pytorch/trial_1/flags_0.json. +I0916 18:10:02.773535 140109961065664 submission_runner.py:359] Starting training loop. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/algorithmic-efficiency/algoperf/workloads/librispeech_conformer/librispeech_pytorch/preprocessor.py:516: UserWarning: Specified kernel cache directory could not be created! This disables kernel caching. Specified directory is /root/.cache/torch/kernels. This warning will appear only once per process. (Triggered internally at /pytorch/aten/src/ATen/native/cuda/jit_utils.cpp:1487.) + spectrum = torch.abs(spectrum) +/algorithmic-efficiency/algoperf/workloads/librispeech_conformer/librispeech_pytorch/preprocessor.py:516: UserWarning: Specified kernel cache directory could not be created! This disables kernel caching. Specified directory is /root/.cache/torch/kernels. This warning will appear only once per process. (Triggered internally at /pytorch/aten/src/ATen/native/cuda/jit_utils.cpp:1487.) + spectrum = torch.abs(spectrum) +I0916 18:10:56.925746 140092989089536 logging_writer.py:48] [0] global_step=0, grad_norm=0.5, loss=33.4458 +I0916 18:10:57.141695 140109961065664 submission.py:307] 0) loss = 33.446, grad_norm = 0.500 +I0916 18:10:57.731207 140109961065664 spec.py:333] Evaluating on the training split. +I0916 18:10:57.732171 140109961065664 input_pipeline.py:19] Loading split = train-clean-100 +I0916 18:10:57.756183 140109961065664 input_pipeline.py:19] Loading split = train-clean-360 +I0916 18:10:57.845455 140109961065664 input_pipeline.py:19] Loading split = train-other-500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 18:11:25.143550 140109961065664 spec.py:346] Evaluating on the validation split. +I0916 18:11:25.144684 140109961065664 input_pipeline.py:19] Loading split = dev-clean +I0916 18:11:25.147782 140109961065664 input_pipeline.py:19] Loading split = dev-other +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 18:11:51.861850 140109961065664 spec.py:363] Evaluating on the test split. +I0916 18:11:51.862965 140109961065664 input_pipeline.py:19] Loading split = test-clean +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 18:12:04.741157 140109961065664 submission_runner.py:516] Time since start: 121.97s, Step: 1, {'train/ctc_loss': 32.54331867872724, 'train/wer': 3.6115683326057315, 'validation/ctc_loss': 31.47319858850713, 'validation/wer': 3.283097571573408, 'validation/num_examples': 5348, 'test/ctc_loss': 31.595735208723145, 'test/wer': 3.6465785144110656, 'test/num_examples': 2472, 'score': 54.36974382400513, 'total_duration': 121.9671106338501, 'accumulated_submission_time': 54.36974382400513, 'accumulated_eval_time': 67.00936770439148, 'accumulated_logging_time': 0} +I0916 18:12:04.835572 140087195002624 logging_writer.py:48] [1] accumulated_eval_time=67.0094, accumulated_logging_time=0, accumulated_submission_time=54.3697, global_step=1, preemption_count=0, score=54.3697, test/ctc_loss=31.5957, test/num_examples=2472, test/wer=3.64658, total_duration=121.967, train/ctc_loss=32.5433, train/wer=3.61157, validation/ctc_loss=31.4732, validation/num_examples=5348, validation/wer=3.2831 +I0916 18:12:06.591432 140087186609920 logging_writer.py:48] [1] global_step=1, grad_norm=0.5, loss=32.8723 +I0916 18:12:06.594582 140109961065664 submission.py:307] 1) loss = 32.872, grad_norm = 0.500 +I0916 18:12:07.375867 140087195002624 logging_writer.py:48] [2] global_step=2, grad_norm=0.5, loss=33.363 +I0916 18:12:07.379040 140109961065664 submission.py:307] 2) loss = 33.363, grad_norm = 0.500 +I0916 18:12:07.927351 140087186609920 logging_writer.py:48] [3] global_step=3, grad_norm=0.5, loss=33.3959 +I0916 18:12:07.930530 140109961065664 submission.py:307] 3) loss = 33.396, grad_norm = 0.500 +I0916 18:12:08.475486 140087195002624 logging_writer.py:48] [4] global_step=4, grad_norm=0.5, loss=32.8386 +I0916 18:12:08.478661 140109961065664 submission.py:307] 4) loss = 32.839, grad_norm = 0.500 +I0916 18:12:09.022360 140087186609920 logging_writer.py:48] [5] global_step=5, grad_norm=0.5, loss=33.0253 +I0916 18:12:09.025453 140109961065664 submission.py:307] 5) loss = 33.025, grad_norm = 0.500 +I0916 18:12:09.578828 140087195002624 logging_writer.py:48] [6] global_step=6, grad_norm=0.5, loss=32.9661 +I0916 18:12:09.582128 140109961065664 submission.py:307] 6) loss = 32.966, grad_norm = 0.500 +I0916 18:12:10.123870 140087186609920 logging_writer.py:48] [7] global_step=7, grad_norm=0.5, loss=31.8327 +I0916 18:12:10.126935 140109961065664 submission.py:307] 7) loss = 31.833, grad_norm = 0.500 +I0916 18:12:10.669741 140087195002624 logging_writer.py:48] [8] global_step=8, grad_norm=0.5, loss=31.9892 +I0916 18:12:10.672911 140109961065664 submission.py:307] 8) loss = 31.989, grad_norm = 0.500 +I0916 18:12:11.221827 140087186609920 logging_writer.py:48] [9] global_step=9, grad_norm=0.5, loss=31.3715 +I0916 18:12:11.224921 140109961065664 submission.py:307] 9) loss = 31.371, grad_norm = 0.500 +I0916 18:12:11.777651 140087195002624 logging_writer.py:48] [10] global_step=10, grad_norm=0.5, loss=31.153 +I0916 18:12:11.780676 140109961065664 submission.py:307] 10) loss = 31.153, grad_norm = 0.500 +I0916 18:12:12.329105 140087186609920 logging_writer.py:48] [11] global_step=11, grad_norm=0.5, loss=30.9121 +I0916 18:12:12.332123 140109961065664 submission.py:307] 11) loss = 30.912, grad_norm = 0.500 +I0916 18:12:12.890544 140087195002624 logging_writer.py:48] [12] global_step=12, grad_norm=0.5, loss=30.9181 +I0916 18:12:12.893628 140109961065664 submission.py:307] 12) loss = 30.918, grad_norm = 0.500 +I0916 18:12:13.441890 140087186609920 logging_writer.py:48] [13] global_step=13, grad_norm=0.5, loss=29.9979 +I0916 18:12:13.444849 140109961065664 submission.py:307] 13) loss = 29.998, grad_norm = 0.500 +I0916 18:12:13.992441 140087195002624 logging_writer.py:48] [14] global_step=14, grad_norm=0.5, loss=29.8516 +I0916 18:12:13.995559 140109961065664 submission.py:307] 14) loss = 29.852, grad_norm = 0.500 +I0916 18:12:14.537038 140087186609920 logging_writer.py:48] [15] global_step=15, grad_norm=0.5, loss=28.5759 +I0916 18:12:14.540111 140109961065664 submission.py:307] 15) loss = 28.576, grad_norm = 0.500 +I0916 18:12:15.083952 140087195002624 logging_writer.py:48] [16] global_step=16, grad_norm=0.5, loss=28.9803 +I0916 18:12:15.086974 140109961065664 submission.py:307] 16) loss = 28.980, grad_norm = 0.500 +I0916 18:12:15.628485 140087186609920 logging_writer.py:48] [17] global_step=17, grad_norm=0.5, loss=28.5636 +I0916 18:12:15.631578 140109961065664 submission.py:307] 17) loss = 28.564, grad_norm = 0.500 +I0916 18:12:16.177556 140087195002624 logging_writer.py:48] [18] global_step=18, grad_norm=0.5, loss=28.1272 +I0916 18:12:16.181074 140109961065664 submission.py:307] 18) loss = 28.127, grad_norm = 0.500 +I0916 18:12:16.734063 140087186609920 logging_writer.py:48] [19] global_step=19, grad_norm=0.5, loss=27.4139 +I0916 18:12:16.736991 140109961065664 submission.py:307] 19) loss = 27.414, grad_norm = 0.500 +I0916 18:12:17.282316 140087195002624 logging_writer.py:48] [20] global_step=20, grad_norm=0.5, loss=26.6362 +I0916 18:12:17.285362 140109961065664 submission.py:307] 20) loss = 26.636, grad_norm = 0.500 +I0916 18:12:17.834186 140087186609920 logging_writer.py:48] [21] global_step=21, grad_norm=0.5, loss=26.5062 +I0916 18:12:17.837347 140109961065664 submission.py:307] 21) loss = 26.506, grad_norm = 0.500 +I0916 18:12:18.385926 140087195002624 logging_writer.py:48] [22] global_step=22, grad_norm=0.5, loss=26.7518 +I0916 18:12:18.388943 140109961065664 submission.py:307] 22) loss = 26.752, grad_norm = 0.500 +I0916 18:12:18.936439 140087186609920 logging_writer.py:48] [23] global_step=23, grad_norm=0.5, loss=25.6558 +I0916 18:12:18.939431 140109961065664 submission.py:307] 23) loss = 25.656, grad_norm = 0.500 +I0916 18:12:19.486013 140087195002624 logging_writer.py:48] [24] global_step=24, grad_norm=0.5, loss=25.4281 +I0916 18:12:19.489008 140109961065664 submission.py:307] 24) loss = 25.428, grad_norm = 0.500 +I0916 18:12:20.034976 140087186609920 logging_writer.py:48] [25] global_step=25, grad_norm=0.5, loss=25.1867 +I0916 18:12:20.038066 140109961065664 submission.py:307] 25) loss = 25.187, grad_norm = 0.500 +I0916 18:12:20.583655 140087195002624 logging_writer.py:48] [26] global_step=26, grad_norm=0.5, loss=24.3208 +I0916 18:12:20.586682 140109961065664 submission.py:307] 26) loss = 24.321, grad_norm = 0.500 +I0916 18:12:21.127753 140087186609920 logging_writer.py:48] [27] global_step=27, grad_norm=0.5, loss=24.1046 +I0916 18:12:21.130797 140109961065664 submission.py:307] 27) loss = 24.105, grad_norm = 0.500 +I0916 18:12:21.682184 140087195002624 logging_writer.py:48] [28] global_step=28, grad_norm=0.5, loss=23.0272 +I0916 18:12:21.685176 140109961065664 submission.py:307] 28) loss = 23.027, grad_norm = 0.500 +I0916 18:12:22.226002 140087186609920 logging_writer.py:48] [29] global_step=29, grad_norm=0.5, loss=22.1892 +I0916 18:12:22.229000 140109961065664 submission.py:307] 29) loss = 22.189, grad_norm = 0.500 +I0916 18:12:22.778469 140087195002624 logging_writer.py:48] [30] global_step=30, grad_norm=0.5, loss=21.7974 +I0916 18:12:22.781595 140109961065664 submission.py:307] 30) loss = 21.797, grad_norm = 0.500 +I0916 18:12:23.321080 140087186609920 logging_writer.py:48] [31] global_step=31, grad_norm=0.5, loss=21.0968 +I0916 18:12:23.324101 140109961065664 submission.py:307] 31) loss = 21.097, grad_norm = 0.500 +I0916 18:12:23.872964 140087195002624 logging_writer.py:48] [32] global_step=32, grad_norm=0.5, loss=20.0563 +I0916 18:12:23.875956 140109961065664 submission.py:307] 32) loss = 20.056, grad_norm = 0.500 +I0916 18:12:24.424965 140087186609920 logging_writer.py:48] [33] global_step=33, grad_norm=0.5, loss=19.8208 +I0916 18:12:24.427934 140109961065664 submission.py:307] 33) loss = 19.821, grad_norm = 0.500 +I0916 18:12:24.973168 140087195002624 logging_writer.py:48] [34] global_step=34, grad_norm=0.5, loss=19.0232 +I0916 18:12:24.976282 140109961065664 submission.py:307] 34) loss = 19.023, grad_norm = 0.500 +I0916 18:12:25.528109 140087186609920 logging_writer.py:48] [35] global_step=35, grad_norm=0.5, loss=18.6837 +I0916 18:12:25.531406 140109961065664 submission.py:307] 35) loss = 18.684, grad_norm = 0.500 +I0916 18:12:26.081274 140087195002624 logging_writer.py:48] [36] global_step=36, grad_norm=0.5, loss=17.7114 +I0916 18:12:26.084325 140109961065664 submission.py:307] 36) loss = 17.711, grad_norm = 0.500 +I0916 18:12:26.626702 140087186609920 logging_writer.py:48] [37] global_step=37, grad_norm=0.5, loss=16.5658 +I0916 18:12:26.629768 140109961065664 submission.py:307] 37) loss = 16.566, grad_norm = 0.500 +I0916 18:12:27.180741 140087195002624 logging_writer.py:48] [38] global_step=38, grad_norm=0.5, loss=16.4611 +I0916 18:12:27.183970 140109961065664 submission.py:307] 38) loss = 16.461, grad_norm = 0.500 +I0916 18:12:27.731287 140087186609920 logging_writer.py:48] [39] global_step=39, grad_norm=0.5, loss=15.9341 +I0916 18:12:27.734389 140109961065664 submission.py:307] 39) loss = 15.934, grad_norm = 0.500 +I0916 18:12:28.280882 140087195002624 logging_writer.py:48] [40] global_step=40, grad_norm=0.5, loss=14.9558 +I0916 18:12:28.284002 140109961065664 submission.py:307] 40) loss = 14.956, grad_norm = 0.500 +I0916 18:12:28.828493 140087186609920 logging_writer.py:48] [41] global_step=41, grad_norm=0.5, loss=14.1912 +I0916 18:12:28.831582 140109961065664 submission.py:307] 41) loss = 14.191, grad_norm = 0.500 +I0916 18:12:29.381525 140087195002624 logging_writer.py:48] [42] global_step=42, grad_norm=0.5, loss=13.5887 +I0916 18:12:29.384690 140109961065664 submission.py:307] 42) loss = 13.589, grad_norm = 0.500 +I0916 18:12:29.936591 140087186609920 logging_writer.py:48] [43] global_step=43, grad_norm=0.5, loss=13.017 +I0916 18:12:29.939555 140109961065664 submission.py:307] 43) loss = 13.017, grad_norm = 0.500 +I0916 18:12:30.483808 140087195002624 logging_writer.py:48] [44] global_step=44, grad_norm=0.5, loss=12.4641 +I0916 18:12:30.486806 140109961065664 submission.py:307] 44) loss = 12.464, grad_norm = 0.500 +I0916 18:12:31.031445 140087186609920 logging_writer.py:48] [45] global_step=45, grad_norm=0.5, loss=11.8128 +I0916 18:12:31.034511 140109961065664 submission.py:307] 45) loss = 11.813, grad_norm = 0.500 +I0916 18:12:31.581750 140087195002624 logging_writer.py:48] [46] global_step=46, grad_norm=0.5, loss=11.2042 +I0916 18:12:31.584806 140109961065664 submission.py:307] 46) loss = 11.204, grad_norm = 0.500 +I0916 18:12:32.131595 140087186609920 logging_writer.py:48] [47] global_step=47, grad_norm=0.5, loss=10.9196 +I0916 18:12:32.134587 140109961065664 submission.py:307] 47) loss = 10.920, grad_norm = 0.500 +I0916 18:12:32.680202 140087195002624 logging_writer.py:48] [48] global_step=48, grad_norm=0.5, loss=10.9876 +I0916 18:12:32.683294 140109961065664 submission.py:307] 48) loss = 10.988, grad_norm = 0.500 +I0916 18:12:33.231607 140087186609920 logging_writer.py:48] [49] global_step=49, grad_norm=0.5, loss=10.5708 +I0916 18:12:33.234710 140109961065664 submission.py:307] 49) loss = 10.571, grad_norm = 0.500 +I0916 18:12:33.777042 140087195002624 logging_writer.py:48] [50] global_step=50, grad_norm=0.5, loss=10.0648 +I0916 18:12:33.780093 140109961065664 submission.py:307] 50) loss = 10.065, grad_norm = 0.500 +I0916 18:12:34.328243 140087186609920 logging_writer.py:48] [51] global_step=51, grad_norm=0.5, loss=9.5508 +I0916 18:12:34.331255 140109961065664 submission.py:307] 51) loss = 9.551, grad_norm = 0.500 +I0916 18:12:34.875115 140087195002624 logging_writer.py:48] [52] global_step=52, grad_norm=0.5, loss=9.38303 +I0916 18:12:34.878137 140109961065664 submission.py:307] 52) loss = 9.383, grad_norm = 0.500 +I0916 18:12:35.428138 140087186609920 logging_writer.py:48] [53] global_step=53, grad_norm=0.5, loss=9.17527 +I0916 18:12:35.431313 140109961065664 submission.py:307] 53) loss = 9.175, grad_norm = 0.500 +I0916 18:12:35.971004 140087195002624 logging_writer.py:48] [54] global_step=54, grad_norm=0.5, loss=9.06011 +I0916 18:12:35.974272 140109961065664 submission.py:307] 54) loss = 9.060, grad_norm = 0.500 +I0916 18:12:36.526288 140087186609920 logging_writer.py:48] [55] global_step=55, grad_norm=0.5, loss=8.94384 +I0916 18:12:36.529536 140109961065664 submission.py:307] 55) loss = 8.944, grad_norm = 0.500 +I0916 18:12:37.074756 140087195002624 logging_writer.py:48] [56] global_step=56, grad_norm=0.5, loss=8.52012 +I0916 18:12:37.077885 140109961065664 submission.py:307] 56) loss = 8.520, grad_norm = 0.500 +I0916 18:12:37.618908 140087186609920 logging_writer.py:48] [57] global_step=57, grad_norm=0.5, loss=8.46748 +I0916 18:12:37.622031 140109961065664 submission.py:307] 57) loss = 8.467, grad_norm = 0.500 +I0916 18:12:38.168297 140087195002624 logging_writer.py:48] [58] global_step=58, grad_norm=0.5, loss=8.03939 +I0916 18:12:38.171314 140109961065664 submission.py:307] 58) loss = 8.039, grad_norm = 0.500 +I0916 18:12:38.717208 140087186609920 logging_writer.py:48] [59] global_step=59, grad_norm=0.5, loss=8.03169 +I0916 18:12:38.720336 140109961065664 submission.py:307] 59) loss = 8.032, grad_norm = 0.500 +I0916 18:12:39.272641 140087195002624 logging_writer.py:48] [60] global_step=60, grad_norm=0.5, loss=7.8031 +I0916 18:12:39.275678 140109961065664 submission.py:307] 60) loss = 7.803, grad_norm = 0.500 +I0916 18:12:39.820654 140087186609920 logging_writer.py:48] [61] global_step=61, grad_norm=0.5, loss=7.60856 +I0916 18:12:39.823842 140109961065664 submission.py:307] 61) loss = 7.609, grad_norm = 0.500 +I0916 18:12:40.369334 140087195002624 logging_writer.py:48] [62] global_step=62, grad_norm=0.5, loss=7.6539 +I0916 18:12:40.372505 140109961065664 submission.py:307] 62) loss = 7.654, grad_norm = 0.500 +I0916 18:12:40.920117 140087186609920 logging_writer.py:48] [63] global_step=63, grad_norm=0.5, loss=7.5183 +I0916 18:12:40.923329 140109961065664 submission.py:307] 63) loss = 7.518, grad_norm = 0.500 +I0916 18:12:41.467074 140087195002624 logging_writer.py:48] [64] global_step=64, grad_norm=0.5, loss=7.34347 +I0916 18:12:41.470131 140109961065664 submission.py:307] 64) loss = 7.343, grad_norm = 0.500 +I0916 18:12:42.010636 140087186609920 logging_writer.py:48] [65] global_step=65, grad_norm=0.5, loss=7.27861 +I0916 18:12:42.013685 140109961065664 submission.py:307] 65) loss = 7.279, grad_norm = 0.500 +I0916 18:12:42.563350 140087195002624 logging_writer.py:48] [66] global_step=66, grad_norm=0.5, loss=7.35773 +I0916 18:12:42.566619 140109961065664 submission.py:307] 66) loss = 7.358, grad_norm = 0.500 +I0916 18:12:43.108328 140087186609920 logging_writer.py:48] [67] global_step=67, grad_norm=0.5, loss=7.29986 +I0916 18:12:43.111443 140109961065664 submission.py:307] 67) loss = 7.300, grad_norm = 0.500 +I0916 18:12:43.658131 140087195002624 logging_writer.py:48] [68] global_step=68, grad_norm=0.5, loss=7.15778 +I0916 18:12:43.661543 140109961065664 submission.py:307] 68) loss = 7.158, grad_norm = 0.500 +I0916 18:12:44.204164 140087186609920 logging_writer.py:48] [69] global_step=69, grad_norm=0.5, loss=7.14655 +I0916 18:12:44.207491 140109961065664 submission.py:307] 69) loss = 7.147, grad_norm = 0.500 +I0916 18:12:44.757632 140087195002624 logging_writer.py:48] [70] global_step=70, grad_norm=0.5, loss=7.03736 +I0916 18:12:44.760766 140109961065664 submission.py:307] 70) loss = 7.037, grad_norm = 0.500 +I0916 18:12:45.302175 140087186609920 logging_writer.py:48] [71] global_step=71, grad_norm=0.5, loss=7.12012 +I0916 18:12:45.305293 140109961065664 submission.py:307] 71) loss = 7.120, grad_norm = 0.500 +I0916 18:12:45.851920 140087195002624 logging_writer.py:48] [72] global_step=72, grad_norm=0.5, loss=7.04406 +I0916 18:12:45.855090 140109961065664 submission.py:307] 72) loss = 7.044, grad_norm = 0.500 +I0916 18:12:46.405373 140087186609920 logging_writer.py:48] [73] global_step=73, grad_norm=0.5, loss=7.03564 +I0916 18:12:46.408394 140109961065664 submission.py:307] 73) loss = 7.036, grad_norm = 0.500 +I0916 18:12:46.950013 140087195002624 logging_writer.py:48] [74] global_step=74, grad_norm=0.5, loss=6.93405 +I0916 18:12:46.953042 140109961065664 submission.py:307] 74) loss = 6.934, grad_norm = 0.500 +I0916 18:12:47.508461 140087186609920 logging_writer.py:48] [75] global_step=75, grad_norm=0.5, loss=6.91863 +I0916 18:12:47.511645 140109961065664 submission.py:307] 75) loss = 6.919, grad_norm = 0.500 +I0916 18:12:48.065047 140087195002624 logging_writer.py:48] [76] global_step=76, grad_norm=0.5, loss=6.86736 +I0916 18:12:48.068058 140109961065664 submission.py:307] 76) loss = 6.867, grad_norm = 0.500 +I0916 18:12:48.617405 140087186609920 logging_writer.py:48] [77] global_step=77, grad_norm=0.5, loss=6.95131 +I0916 18:12:48.620342 140109961065664 submission.py:307] 77) loss = 6.951, grad_norm = 0.500 +I0916 18:12:49.164760 140087195002624 logging_writer.py:48] [78] global_step=78, grad_norm=0.5, loss=6.87848 +I0916 18:12:49.167762 140109961065664 submission.py:307] 78) loss = 6.878, grad_norm = 0.500 +I0916 18:12:49.715335 140087186609920 logging_writer.py:48] [79] global_step=79, grad_norm=0.5, loss=6.81268 +I0916 18:12:49.718436 140109961065664 submission.py:307] 79) loss = 6.813, grad_norm = 0.500 +I0916 18:12:50.267109 140087195002624 logging_writer.py:48] [80] global_step=80, grad_norm=0.5, loss=6.77503 +I0916 18:12:50.270277 140109961065664 submission.py:307] 80) loss = 6.775, grad_norm = 0.500 +I0916 18:12:50.817227 140087186609920 logging_writer.py:48] [81] global_step=81, grad_norm=0.5, loss=6.67952 +I0916 18:12:50.820400 140109961065664 submission.py:307] 81) loss = 6.680, grad_norm = 0.500 +I0916 18:12:51.361932 140087195002624 logging_writer.py:48] [82] global_step=82, grad_norm=0.5, loss=6.73085 +I0916 18:12:51.364939 140109961065664 submission.py:307] 82) loss = 6.731, grad_norm = 0.500 +I0916 18:12:51.907779 140087186609920 logging_writer.py:48] [83] global_step=83, grad_norm=0.5, loss=6.71836 +I0916 18:12:51.910872 140109961065664 submission.py:307] 83) loss = 6.718, grad_norm = 0.500 +I0916 18:12:52.463570 140087195002624 logging_writer.py:48] [84] global_step=84, grad_norm=0.5, loss=6.65467 +I0916 18:12:52.466636 140109961065664 submission.py:307] 84) loss = 6.655, grad_norm = 0.500 +I0916 18:12:53.013191 140087186609920 logging_writer.py:48] [85] global_step=85, grad_norm=0.5, loss=6.57361 +I0916 18:12:53.016291 140109961065664 submission.py:307] 85) loss = 6.574, grad_norm = 0.500 +I0916 18:12:53.564555 140087195002624 logging_writer.py:48] [86] global_step=86, grad_norm=0.5, loss=6.63613 +I0916 18:12:53.567737 140109961065664 submission.py:307] 86) loss = 6.636, grad_norm = 0.500 +I0916 18:12:54.117468 140087186609920 logging_writer.py:48] [87] global_step=87, grad_norm=0.5, loss=6.57787 +I0916 18:12:54.120408 140109961065664 submission.py:307] 87) loss = 6.578, grad_norm = 0.500 +I0916 18:12:54.678016 140087195002624 logging_writer.py:48] [88] global_step=88, grad_norm=0.5, loss=6.5355 +I0916 18:12:54.681066 140109961065664 submission.py:307] 88) loss = 6.536, grad_norm = 0.500 +I0916 18:12:55.224258 140087186609920 logging_writer.py:48] [89] global_step=89, grad_norm=0.5, loss=6.53852 +I0916 18:12:55.227237 140109961065664 submission.py:307] 89) loss = 6.539, grad_norm = 0.500 +I0916 18:12:55.781403 140087195002624 logging_writer.py:48] [90] global_step=90, grad_norm=0.5, loss=6.46552 +I0916 18:12:55.784550 140109961065664 submission.py:307] 90) loss = 6.466, grad_norm = 0.500 +I0916 18:12:56.332598 140087186609920 logging_writer.py:48] [91] global_step=91, grad_norm=0.5, loss=6.51518 +I0916 18:12:56.335608 140109961065664 submission.py:307] 91) loss = 6.515, grad_norm = 0.500 +I0916 18:12:56.891229 140087195002624 logging_writer.py:48] [92] global_step=92, grad_norm=0.5, loss=6.50284 +I0916 18:12:56.894268 140109961065664 submission.py:307] 92) loss = 6.503, grad_norm = 0.500 +I0916 18:12:57.443939 140087186609920 logging_writer.py:48] [93] global_step=93, grad_norm=0.5, loss=6.43526 +I0916 18:12:57.447015 140109961065664 submission.py:307] 93) loss = 6.435, grad_norm = 0.500 +I0916 18:12:58.001436 140087195002624 logging_writer.py:48] [94] global_step=94, grad_norm=0.5, loss=6.37654 +I0916 18:12:58.004537 140109961065664 submission.py:307] 94) loss = 6.377, grad_norm = 0.500 +I0916 18:12:58.556436 140087186609920 logging_writer.py:48] [95] global_step=95, grad_norm=0.5, loss=6.34783 +I0916 18:12:58.559534 140109961065664 submission.py:307] 95) loss = 6.348, grad_norm = 0.500 +I0916 18:12:59.109855 140087195002624 logging_writer.py:48] [96] global_step=96, grad_norm=0.5, loss=6.39257 +I0916 18:12:59.112868 140109961065664 submission.py:307] 96) loss = 6.393, grad_norm = 0.500 +I0916 18:12:59.658155 140087186609920 logging_writer.py:48] [97] global_step=97, grad_norm=0.5, loss=6.31858 +I0916 18:12:59.661388 140109961065664 submission.py:307] 97) loss = 6.319, grad_norm = 0.500 +I0916 18:13:00.212704 140087195002624 logging_writer.py:48] [98] global_step=98, grad_norm=0.5, loss=6.30538 +I0916 18:13:00.215799 140109961065664 submission.py:307] 98) loss = 6.305, grad_norm = 0.500 +I0916 18:13:00.762246 140087186609920 logging_writer.py:48] [99] global_step=99, grad_norm=0.5, loss=6.26942 +I0916 18:13:00.765244 140109961065664 submission.py:307] 99) loss = 6.269, grad_norm = 0.500 +I0916 18:13:01.309019 140087195002624 logging_writer.py:48] [100] global_step=100, grad_norm=0.5, loss=6.30538 +I0916 18:13:01.312143 140109961065664 submission.py:307] 100) loss = 6.305, grad_norm = 0.500 +I0916 18:16:41.466311 140087186609920 logging_writer.py:48] [500] global_step=500, grad_norm=0.5, loss=4.64759 +I0916 18:16:41.470275 140109961065664 submission.py:307] 500) loss = 4.648, grad_norm = 0.500 +I0916 18:23:38.281201 140087195002624 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.5, loss=3.06197 +I0916 18:23:38.284799 140109961065664 submission.py:307] 1000) loss = 3.062, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 18:28:24.930496 140087195002624 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.5, loss=2.64044 +I0916 18:28:24.937575 140109961065664 submission.py:307] 1500) loss = 2.640, grad_norm = 0.500 +I0916 18:36:12.649199 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 18:36:25.865228 140109961065664 spec.py:346] Evaluating on the validation split. +I0916 18:36:45.400866 140109961065664 spec.py:363] Evaluating on the test split. +I0916 18:36:55.728207 140109961065664 submission_runner.py:516] Time since start: 1612.95s, Step: 1991, {'train/ctc_loss': 6.037100156652686, 'train/wer': 0.9414253453414609, 'validation/ctc_loss': 5.9608374303044, 'validation/wer': 0.8964901269733984, 'validation/num_examples': 5348, 'test/ctc_loss': 5.847129653526034, 'test/wer': 0.8991123839701014, 'test/num_examples': 2472, 'score': 1499.6107699871063, 'total_duration': 1612.9545664787292, 'accumulated_submission_time': 1499.6107699871063, 'accumulated_eval_time': 110.08822107315063, 'accumulated_logging_time': 0.10763931274414062} +I0916 18:36:55.930017 140087195002624 logging_writer.py:48] [1991] accumulated_eval_time=110.088, accumulated_logging_time=0.107639, accumulated_submission_time=1499.61, global_step=1991, preemption_count=0, score=1499.61, test/ctc_loss=5.84713, test/num_examples=2472, test/wer=0.899112, total_duration=1612.95, train/ctc_loss=6.0371, train/wer=0.941425, validation/ctc_loss=5.96084, validation/num_examples=5348, validation/wer=0.89649 +I0916 18:37:03.025831 140087186609920 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.5, loss=2.42904 +I0916 18:37:03.029104 140109961065664 submission.py:307] 2000) loss = 2.429, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 18:42:08.863101 140087113082624 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.5, loss=2.22706 +I0916 18:42:08.873274 140109961065664 submission.py:307] 2500) loss = 2.227, grad_norm = 0.500 +I0916 18:49:47.660837 140087104689920 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.5, loss=2.10497 +I0916 18:49:47.664771 140109961065664 submission.py:307] 3000) loss = 2.105, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 18:55:18.487609 140087195002624 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.5, loss=2.00826 +I0916 18:55:18.494603 140109961065664 submission.py:307] 3500) loss = 2.008, grad_norm = 0.500 +I0916 19:01:03.975427 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 19:01:22.143468 140109961065664 spec.py:346] Evaluating on the validation split. +I0916 19:01:42.219096 140109961065664 spec.py:363] Evaluating on the test split. +I0916 19:01:52.522760 140109961065664 submission_runner.py:516] Time since start: 3109.75s, Step: 3933, {'train/ctc_loss': 1.6419855517374973, 'train/wer': 0.4234104591438011, 'validation/ctc_loss': 1.819571591822383, 'validation/wer': 0.43858446386327427, 'validation/num_examples': 5348, 'test/ctc_loss': 1.2938803511516872, 'test/wer': 0.349074807547783, 'test/num_examples': 2472, 'score': 2945.1600375175476, 'total_duration': 3109.749101638794, 'accumulated_submission_time': 2945.1600375175476, 'accumulated_eval_time': 158.6353361606598, 'accumulated_logging_time': 0.3192300796508789} +I0916 19:01:52.601986 140087195002624 logging_writer.py:48] [3933] accumulated_eval_time=158.635, accumulated_logging_time=0.31923, accumulated_submission_time=2945.16, global_step=3933, preemption_count=0, score=2945.16, test/ctc_loss=1.29388, test/num_examples=2472, test/wer=0.349075, total_duration=3109.75, train/ctc_loss=1.64199, train/wer=0.42341, validation/ctc_loss=1.81957, validation/num_examples=5348, validation/wer=0.438584 +I0916 19:03:03.321179 140087186609920 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.5, loss=1.93263 +I0916 19:03:03.324830 140109961065664 submission.py:307] 4000) loss = 1.933, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 19:08:48.288359 140087195002624 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.5, loss=1.8837 +I0916 19:08:48.295222 140109961065664 submission.py:307] 4500) loss = 1.884, grad_norm = 0.500 +I0916 19:15:41.076898 140087186609920 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.5, loss=1.88535 +I0916 19:15:41.080799 140109961065664 submission.py:307] 5000) loss = 1.885, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 19:21:46.521195 140087195002624 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.5, loss=1.84784 +I0916 19:21:46.528305 140109961065664 submission.py:307] 5500) loss = 1.848, grad_norm = 0.500 +I0916 19:26:00.491144 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 19:26:19.325731 140109961065664 spec.py:346] Evaluating on the validation split. +I0916 19:26:39.310572 140109961065664 spec.py:363] Evaluating on the test split. +I0916 19:26:50.598206 140109961065664 submission_runner.py:516] Time since start: 4607.82s, Step: 5867, {'train/ctc_loss': 0.5884155970488655, 'train/wer': 0.19020054866579356, 'validation/ctc_loss': 0.8283974446516476, 'validation/wer': 0.23493458214647805, 'validation/num_examples': 5348, 'test/ctc_loss': 0.512976744467385, 'test/wer': 0.1638128897284342, 'test/num_examples': 2472, 'score': 4390.693488121033, 'total_duration': 4607.824376344681, 'accumulated_submission_time': 4390.693488121033, 'accumulated_eval_time': 208.74199318885803, 'accumulated_logging_time': 0.40804481506347656} +I0916 19:26:50.675309 140087195002624 logging_writer.py:48] [5867] accumulated_eval_time=208.742, accumulated_logging_time=0.408045, accumulated_submission_time=4390.69, global_step=5867, preemption_count=0, score=4390.69, test/ctc_loss=0.512977, test/num_examples=2472, test/wer=0.163813, total_duration=4607.82, train/ctc_loss=0.588416, train/wer=0.190201, validation/ctc_loss=0.828397, validation/num_examples=5348, validation/wer=0.234935 +I0916 19:29:13.612985 140087186609920 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.5, loss=1.7736 +I0916 19:29:13.616657 140109961065664 submission.py:307] 6000) loss = 1.774, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 19:35:32.240906 140087195002624 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.5, loss=1.72951 +I0916 19:35:32.248065 140109961065664 submission.py:307] 6500) loss = 1.730, grad_norm = 0.500 +I0916 19:41:47.092361 140087186609920 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.5, loss=1.69544 +I0916 19:41:47.096176 140109961065664 submission.py:307] 7000) loss = 1.695, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 19:48:21.707862 140087195002624 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.5, loss=1.71441 +I0916 19:48:21.714993 140109961065664 submission.py:307] 7500) loss = 1.714, grad_norm = 0.500 +I0916 19:50:58.694969 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 19:51:15.332918 140109961065664 spec.py:346] Evaluating on the validation split. +I0916 19:51:35.651342 140109961065664 spec.py:363] Evaluating on the test split. +I0916 19:51:46.008849 140109961065664 submission_runner.py:516] Time since start: 6103.24s, Step: 7769, {'train/ctc_loss': 0.4873721933060454, 'train/wer': 0.16171627834279215, 'validation/ctc_loss': 0.7424400288200723, 'validation/wer': 0.2120600589002076, 'validation/num_examples': 5348, 'test/ctc_loss': 0.4462445157752113, 'test/wer': 0.14461844697662138, 'test/num_examples': 2472, 'score': 5836.115516662598, 'total_duration': 6103.2351706027985, 'accumulated_submission_time': 5836.115516662598, 'accumulated_eval_time': 256.05564308166504, 'accumulated_logging_time': 0.49970126152038574} +I0916 19:51:46.181147 140087195002624 logging_writer.py:48] [7769] accumulated_eval_time=256.056, accumulated_logging_time=0.499701, accumulated_submission_time=5836.12, global_step=7769, preemption_count=0, score=5836.12, test/ctc_loss=0.446245, test/num_examples=2472, test/wer=0.144618, total_duration=6103.24, train/ctc_loss=0.487372, train/wer=0.161716, validation/ctc_loss=0.74244, validation/num_examples=5348, validation/wer=0.21206 +I0916 19:55:21.137408 140087186609920 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.5, loss=1.56077 +I0916 19:55:21.141041 140109961065664 submission.py:307] 8000) loss = 1.561, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 20:02:15.229484 140087195002624 logging_writer.py:48] [8500] global_step=8500, grad_norm=0.5, loss=1.55321 +I0916 20:02:15.236678 140109961065664 submission.py:307] 8500) loss = 1.553, grad_norm = 0.500 +I0916 20:07:58.228713 140087186609920 logging_writer.py:48] [9000] global_step=9000, grad_norm=0.5, loss=1.65831 +I0916 20:07:58.232606 140109961065664 submission.py:307] 9000) loss = 1.658, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 20:15:05.433575 140087195002624 logging_writer.py:48] [9500] global_step=9500, grad_norm=0.5, loss=1.5993 +I0916 20:15:05.440623 140109961065664 submission.py:307] 9500) loss = 1.599, grad_norm = 0.500 +I0916 20:15:54.295756 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 20:16:09.245176 140109961065664 spec.py:346] Evaluating on the validation split. +I0916 20:16:29.113103 140109961065664 spec.py:363] Evaluating on the test split. +I0916 20:16:39.819638 140109961065664 submission_runner.py:516] Time since start: 7597.05s, Step: 9590, {'train/ctc_loss': 0.4252956714167804, 'train/wer': 0.1424644687697059, 'validation/ctc_loss': 0.6756669056566456, 'validation/wer': 0.19341476367498672, 'validation/num_examples': 5348, 'test/ctc_loss': 0.39390283244080265, 'test/wer': 0.128998842239961, 'test/num_examples': 2472, 'score': 7281.850630283356, 'total_duration': 7597.046008348465, 'accumulated_submission_time': 7281.850630283356, 'accumulated_eval_time': 301.57932806015015, 'accumulated_logging_time': 0.6819305419921875} +I0916 20:16:39.876712 140087195002624 logging_writer.py:48] [9590] accumulated_eval_time=301.579, accumulated_logging_time=0.681931, accumulated_submission_time=7281.85, global_step=9590, preemption_count=0, score=7281.85, test/ctc_loss=0.393903, test/num_examples=2472, test/wer=0.128999, total_duration=7597.05, train/ctc_loss=0.425296, train/wer=0.142464, validation/ctc_loss=0.675667, validation/num_examples=5348, validation/wer=0.193415 +I0916 20:21:45.208678 140087186609920 logging_writer.py:48] [10000] global_step=10000, grad_norm=0.5, loss=1.68571 +I0916 20:21:45.212584 140109961065664 submission.py:307] 10000) loss = 1.686, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 20:29:08.147101 140087195002624 logging_writer.py:48] [10500] global_step=10500, grad_norm=0.5, loss=1.58566 +I0916 20:29:08.154248 140109961065664 submission.py:307] 10500) loss = 1.586, grad_norm = 0.500 +I0916 20:34:30.587596 140087186609920 logging_writer.py:48] [11000] global_step=11000, grad_norm=0.5, loss=1.51798 +I0916 20:34:30.591294 140109961065664 submission.py:307] 11000) loss = 1.518, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 20:40:48.058598 140109961065664 spec.py:333] Evaluating on the training split. +I0916 20:41:02.749731 140109961065664 spec.py:346] Evaluating on the validation split. +I0916 20:41:22.698508 140109961065664 spec.py:363] Evaluating on the test split. +I0916 20:41:33.099156 140109961065664 submission_runner.py:516] Time since start: 9090.33s, Step: 11351, {'train/ctc_loss': 0.38339447315006653, 'train/wer': 0.12967484275712646, 'validation/ctc_loss': 0.6379087945172794, 'validation/wer': 0.18445420750253463, 'validation/num_examples': 5348, 'test/ctc_loss': 0.36346412470159367, 'test/wer': 0.1171775028944001, 'test/num_examples': 2472, 'score': 8727.689821004868, 'total_duration': 9090.325487852097, 'accumulated_submission_time': 8727.689821004868, 'accumulated_eval_time': 346.61975502967834, 'accumulated_logging_time': 0.7487177848815918} +I0916 20:41:33.149096 140087195002624 logging_writer.py:48] [11351] accumulated_eval_time=346.62, accumulated_logging_time=0.748718, accumulated_submission_time=8727.69, global_step=11351, preemption_count=0, score=8727.69, test/ctc_loss=0.363464, test/num_examples=2472, test/wer=0.117178, total_duration=9090.33, train/ctc_loss=0.383394, train/wer=0.129675, validation/ctc_loss=0.637909, validation/num_examples=5348, validation/wer=0.184454 +I0916 20:42:55.353090 140087186609920 logging_writer.py:48] [11500] global_step=11500, grad_norm=0.5, loss=1.50288 +I0916 20:42:55.356727 140109961065664 submission.py:307] 11500) loss = 1.503, grad_norm = 0.500 +I0916 20:48:16.566365 140087195002624 logging_writer.py:48] [12000] global_step=12000, grad_norm=0.5, loss=1.47386 +I0916 20:48:16.569994 140109961065664 submission.py:307] 12000) loss = 1.474, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 20:56:00.948086 140087113082624 logging_writer.py:48] [12500] global_step=12500, grad_norm=0.5, loss=1.51324 +I0916 20:56:00.956786 140109961065664 submission.py:307] 12500) loss = 1.513, grad_norm = 0.500 +I0916 21:01:03.630248 140087104689920 logging_writer.py:48] [13000] global_step=13000, grad_norm=0.5, loss=1.5206 +I0916 21:01:03.634029 140109961065664 submission.py:307] 13000) loss = 1.521, grad_norm = 0.500 +I0916 21:05:41.455753 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 21:05:56.894614 140109961065664 spec.py:346] Evaluating on the validation split. +I0916 21:06:16.748930 140109961065664 spec.py:363] Evaluating on the test split. +I0916 21:06:27.289433 140109961065664 submission_runner.py:516] Time since start: 10584.52s, Step: 13269, {'train/ctc_loss': 0.35512769088887763, 'train/wer': 0.12238804361300198, 'validation/ctc_loss': 0.6118201343366988, 'validation/wer': 0.17457635301501473, 'validation/num_examples': 5348, 'test/ctc_loss': 0.33827091667204334, 'test/wer': 0.11102309426604107, 'test/num_examples': 2472, 'score': 10173.496448755264, 'total_duration': 10584.515745162964, 'accumulated_submission_time': 10173.496448755264, 'accumulated_eval_time': 392.4531898498535, 'accumulated_logging_time': 0.8087592124938965} +I0916 21:06:27.486925 140087195002624 logging_writer.py:48] [13269] accumulated_eval_time=392.453, accumulated_logging_time=0.808759, accumulated_submission_time=10173.5, global_step=13269, preemption_count=0, score=10173.5, test/ctc_loss=0.338271, test/num_examples=2472, test/wer=0.111023, total_duration=10584.5, train/ctc_loss=0.355128, train/wer=0.122388, validation/ctc_loss=0.61182, validation/num_examples=5348, validation/wer=0.174576 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 21:09:45.724510 140087195002624 logging_writer.py:48] [13500] global_step=13500, grad_norm=0.5, loss=1.37534 +I0916 21:09:45.731490 140109961065664 submission.py:307] 13500) loss = 1.375, grad_norm = 0.500 +I0916 21:14:49.123866 140087186609920 logging_writer.py:48] [14000] global_step=14000, grad_norm=0.5, loss=1.39868 +I0916 21:14:49.127690 140109961065664 submission.py:307] 14000) loss = 1.399, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 21:22:43.365637 140087195002624 logging_writer.py:48] [14500] global_step=14500, grad_norm=0.5, loss=1.45877 +I0916 21:22:43.372887 140109961065664 submission.py:307] 14500) loss = 1.459, grad_norm = 0.500 +I0916 21:27:39.885377 140087186609920 logging_writer.py:48] [15000] global_step=15000, grad_norm=0.5, loss=1.45926 +I0916 21:27:39.889184 140109961065664 submission.py:307] 15000) loss = 1.459, grad_norm = 0.500 +I0916 21:30:35.533857 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 21:30:53.730270 140109961065664 spec.py:346] Evaluating on the validation split. +I0916 21:31:13.551105 140109961065664 spec.py:363] Evaluating on the test split. +I0916 21:31:23.998073 140109961065664 submission_runner.py:516] Time since start: 12081.22s, Step: 15193, {'train/ctc_loss': 0.3283039978018091, 'train/wer': 0.11291843851224258, 'validation/ctc_loss': 0.5931179421432339, 'validation/wer': 0.16854149567904214, 'validation/num_examples': 5348, 'test/ctc_loss': 0.32434942879379314, 'test/wer': 0.10529522880994455, 'test/num_examples': 2472, 'score': 11619.135604143143, 'total_duration': 12081.224436283112, 'accumulated_submission_time': 11619.135604143143, 'accumulated_eval_time': 440.9172077178955, 'accumulated_logging_time': 1.01652193069458} +I0916 21:31:24.079372 140087195002624 logging_writer.py:48] [15193] accumulated_eval_time=440.917, accumulated_logging_time=1.01652, accumulated_submission_time=11619.1, global_step=15193, preemption_count=0, score=11619.1, test/ctc_loss=0.324349, test/num_examples=2472, test/wer=0.105295, total_duration=12081.2, train/ctc_loss=0.328304, train/wer=0.112918, validation/ctc_loss=0.593118, validation/num_examples=5348, validation/wer=0.168541 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 21:36:39.469419 140087195002624 logging_writer.py:48] [15500] global_step=15500, grad_norm=0.5, loss=1.3529 +I0916 21:36:39.476499 140109961065664 submission.py:307] 15500) loss = 1.353, grad_norm = 0.500 +I0916 21:41:23.138920 140087186609920 logging_writer.py:48] [16000] global_step=16000, grad_norm=0.5, loss=1.38706 +I0916 21:41:23.142918 140109961065664 submission.py:307] 16000) loss = 1.387, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 21:49:27.961204 140087195002624 logging_writer.py:48] [16500] global_step=16500, grad_norm=0.5, loss=1.42073 +I0916 21:49:27.968323 140109961065664 submission.py:307] 16500) loss = 1.421, grad_norm = 0.500 +I0916 21:54:03.177615 140087186609920 logging_writer.py:48] [17000] global_step=17000, grad_norm=0.5, loss=1.37118 +I0916 21:54:03.181572 140109961065664 submission.py:307] 17000) loss = 1.371, grad_norm = 0.500 +I0916 21:55:32.822628 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 21:55:50.024690 140109961065664 spec.py:346] Evaluating on the validation split. +I0916 21:56:09.848051 140109961065664 spec.py:363] Evaluating on the test split. +I0916 21:56:20.315183 140109961065664 submission_runner.py:516] Time since start: 13577.54s, Step: 17119, {'train/ctc_loss': 0.3036698365674533, 'train/wer': 0.10576099083221498, 'validation/ctc_loss': 0.5600870449379647, 'validation/wer': 0.1598416453435041, 'validation/num_examples': 5348, 'test/ctc_loss': 0.30678115826021035, 'test/wer': 0.09930331281863791, 'test/num_examples': 2472, 'score': 13065.391492843628, 'total_duration': 13577.54152393341, 'accumulated_submission_time': 13065.391492843628, 'accumulated_eval_time': 488.40959787368774, 'accumulated_logging_time': 1.1076455116271973} +I0916 21:56:20.387634 140087195002624 logging_writer.py:48] [17119] accumulated_eval_time=488.41, accumulated_logging_time=1.10765, accumulated_submission_time=13065.4, global_step=17119, preemption_count=0, score=13065.4, test/ctc_loss=0.306781, test/num_examples=2472, test/wer=0.0993033, total_duration=13577.5, train/ctc_loss=0.30367, train/wer=0.105761, validation/ctc_loss=0.560087, validation/num_examples=5348, validation/wer=0.159842 +I0916 22:03:10.977513 140087186609920 logging_writer.py:48] [17500] global_step=17500, grad_norm=0.5, loss=1.45792 +I0916 22:03:10.981376 140109961065664 submission.py:307] 17500) loss = 1.458, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 22:07:50.020411 140087195002624 logging_writer.py:48] [18000] global_step=18000, grad_norm=0.5, loss=1.37395 +I0916 22:07:50.027199 140109961065664 submission.py:307] 18000) loss = 1.374, grad_norm = 0.500 +I0916 22:15:39.291338 140087186609920 logging_writer.py:48] [18500] global_step=18500, grad_norm=0.5, loss=1.35524 +I0916 22:15:39.295083 140109961065664 submission.py:307] 18500) loss = 1.355, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 22:20:28.461816 140109961065664 spec.py:333] Evaluating on the training split. +I0916 22:20:44.186938 140109961065664 spec.py:346] Evaluating on the validation split. +I0916 22:21:04.058147 140109961065664 spec.py:363] Evaluating on the test split. +I0916 22:21:14.540283 140109961065664 submission_runner.py:516] Time since start: 15071.77s, Step: 18990, {'train/ctc_loss': 0.2967907781258949, 'train/wer': 0.10316318226160255, 'validation/ctc_loss': 0.5587584725675607, 'validation/wer': 0.15916574132187514, 'validation/num_examples': 5348, 'test/ctc_loss': 0.29955332299664494, 'test/wer': 0.09806430646111348, 'test/num_examples': 2472, 'score': 14511.024166345596, 'total_duration': 15071.766645908356, 'accumulated_submission_time': 14511.024166345596, 'accumulated_eval_time': 534.4881474971771, 'accumulated_logging_time': 1.189727783203125} +I0916 22:21:14.604496 140087195002624 logging_writer.py:48] [18990] accumulated_eval_time=534.488, accumulated_logging_time=1.18973, accumulated_submission_time=14511, global_step=18990, preemption_count=0, score=14511, test/ctc_loss=0.299553, test/num_examples=2472, test/wer=0.0980643, total_duration=15071.8, train/ctc_loss=0.296791, train/wer=0.103163, validation/ctc_loss=0.558758, validation/num_examples=5348, validation/wer=0.159166 +I0916 22:21:22.244594 140087186609920 logging_writer.py:48] [19000] global_step=19000, grad_norm=0.5, loss=1.33257 +I0916 22:21:22.247876 140109961065664 submission.py:307] 19000) loss = 1.333, grad_norm = 0.500 +I0916 22:29:18.163352 140087195002624 logging_writer.py:48] [19500] global_step=19500, grad_norm=0.5, loss=1.45576 +I0916 22:29:18.167100 140109961065664 submission.py:307] 19500) loss = 1.456, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 22:34:32.645886 140087195002624 logging_writer.py:48] [20000] global_step=20000, grad_norm=0.5, loss=1.3754 +I0916 22:34:32.652859 140109961065664 submission.py:307] 20000) loss = 1.375, grad_norm = 0.500 +I0916 22:41:53.719961 140087186609920 logging_writer.py:48] [20500] global_step=20500, grad_norm=0.5, loss=1.39109 +I0916 22:41:53.723837 140109961065664 submission.py:307] 20500) loss = 1.391, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 22:45:22.873360 140109961065664 spec.py:333] Evaluating on the training split. +I0916 22:45:38.481454 140109961065664 spec.py:346] Evaluating on the validation split. +I0916 22:45:58.373758 140109961065664 spec.py:363] Evaluating on the test split. +I0916 22:46:08.660260 140109961065664 submission_runner.py:516] Time since start: 16565.89s, Step: 20770, {'train/ctc_loss': 0.2835062776879411, 'train/wer': 0.0986736085285732, 'validation/ctc_loss': 0.5481313652175005, 'validation/wer': 0.1545985612900111, 'validation/num_examples': 5348, 'test/ctc_loss': 0.29365324476579135, 'test/wer': 0.09554567058680154, 'test/num_examples': 2472, 'score': 15956.933192014694, 'total_duration': 16565.886620759964, 'accumulated_submission_time': 15956.933192014694, 'accumulated_eval_time': 580.2749526500702, 'accumulated_logging_time': 1.2638778686523438} +I0916 22:46:08.875057 140087195002624 logging_writer.py:48] [20770] accumulated_eval_time=580.275, accumulated_logging_time=1.26388, accumulated_submission_time=15956.9, global_step=20770, preemption_count=0, score=15956.9, test/ctc_loss=0.293653, test/num_examples=2472, test/wer=0.0955457, total_duration=16565.9, train/ctc_loss=0.283506, train/wer=0.0986736, validation/ctc_loss=0.548131, validation/num_examples=5348, validation/wer=0.154599 +I0916 22:48:15.482375 140087186609920 logging_writer.py:48] [21000] global_step=21000, grad_norm=0.5, loss=1.3774 +I0916 22:48:15.486050 140109961065664 submission.py:307] 21000) loss = 1.377, grad_norm = 0.500 +I0916 22:55:33.122776 140087195002624 logging_writer.py:48] [21500] global_step=21500, grad_norm=0.5, loss=1.33048 +I0916 22:55:33.269998 140109961065664 submission.py:307] 21500) loss = 1.330, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 23:01:24.045986 140087195002624 logging_writer.py:48] [22000] global_step=22000, grad_norm=0.5, loss=1.30799 +I0916 23:01:24.054826 140109961065664 submission.py:307] 22000) loss = 1.308, grad_norm = 0.500 +I0916 23:08:13.134290 140087186609920 logging_writer.py:48] [22500] global_step=22500, grad_norm=0.5, loss=1.33827 +I0916 23:08:13.138145 140109961065664 submission.py:307] 22500) loss = 1.338, grad_norm = 0.500 +I0916 23:10:17.626431 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 23:10:33.388386 140109961065664 spec.py:346] Evaluating on the validation split. +I0916 23:10:54.055569 140109961065664 spec.py:363] Evaluating on the test split. +I0916 23:11:04.708451 140109961065664 submission_runner.py:516] Time since start: 18061.93s, Step: 22608, {'train/ctc_loss': 0.2634278618592821, 'train/wer': 0.09291207873192447, 'validation/ctc_loss': 0.5222968526314798, 'validation/wer': 0.148640950127939, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2798856982708562, 'test/wer': 0.09060995673633539, 'test/num_examples': 2472, 'score': 17403.099194288254, 'total_duration': 18061.93473124504, 'accumulated_submission_time': 17403.099194288254, 'accumulated_eval_time': 627.3567457199097, 'accumulated_logging_time': 1.4887628555297852} +I0916 23:11:04.764576 140087195002624 logging_writer.py:48] [22608] accumulated_eval_time=627.357, accumulated_logging_time=1.48876, accumulated_submission_time=17403.1, global_step=22608, preemption_count=0, score=17403.1, test/ctc_loss=0.279886, test/num_examples=2472, test/wer=0.09061, total_duration=18061.9, train/ctc_loss=0.263428, train/wer=0.0929121, validation/ctc_loss=0.522297, validation/num_examples=5348, validation/wer=0.148641 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 23:15:11.187780 140087195002624 logging_writer.py:48] [23000] global_step=23000, grad_norm=0.5, loss=1.30225 +I0916 23:15:11.195407 140109961065664 submission.py:307] 23000) loss = 1.302, grad_norm = 0.500 +I0916 23:21:50.352147 140087186609920 logging_writer.py:48] [23500] global_step=23500, grad_norm=0.5, loss=1.29382 +I0916 23:21:50.355928 140109961065664 submission.py:307] 23500) loss = 1.294, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 23:28:16.428102 140087195002624 logging_writer.py:48] [24000] global_step=24000, grad_norm=0.5, loss=1.29155 +I0916 23:28:16.435130 140109961065664 submission.py:307] 24000) loss = 1.292, grad_norm = 0.500 +I0916 23:34:36.678757 140087186609920 logging_writer.py:48] [24500] global_step=24500, grad_norm=0.5, loss=1.32228 +I0916 23:34:36.682705 140109961065664 submission.py:307] 24500) loss = 1.322, grad_norm = 0.500 +I0916 23:35:12.580346 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 23:35:27.704670 140109961065664 spec.py:346] Evaluating on the validation split. +I0916 23:35:47.700914 140109961065664 spec.py:363] Evaluating on the test split. +I0916 23:35:58.279770 140109961065664 submission_runner.py:516] Time since start: 19555.51s, Step: 24531, {'train/ctc_loss': 0.24694506207152123, 'train/wer': 0.08706970427021521, 'validation/ctc_loss': 0.5107892791214587, 'validation/wer': 0.14457587022642784, 'validation/num_examples': 5348, 'test/ctc_loss': 0.27159272372411125, 'test/wer': 0.08733979241565616, 'test/num_examples': 2472, 'score': 18848.515809059143, 'total_duration': 19555.506080389023, 'accumulated_submission_time': 18848.515809059143, 'accumulated_eval_time': 673.05593085289, 'accumulated_logging_time': 1.5548083782196045} +I0916 23:35:58.336251 140087195002624 logging_writer.py:48] [24531] accumulated_eval_time=673.056, accumulated_logging_time=1.55481, accumulated_submission_time=18848.5, global_step=24531, preemption_count=0, score=18848.5, test/ctc_loss=0.271593, test/num_examples=2472, test/wer=0.0873398, total_duration=19555.5, train/ctc_loss=0.246945, train/wer=0.0870697, validation/ctc_loss=0.510789, validation/num_examples=5348, validation/wer=0.144576 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 23:42:05.682437 140087195002624 logging_writer.py:48] [25000] global_step=25000, grad_norm=0.5, loss=1.30206 +I0916 23:42:05.689285 140109961065664 submission.py:307] 25000) loss = 1.302, grad_norm = 0.500 +I0916 23:48:04.703043 140087186609920 logging_writer.py:48] [25500] global_step=25500, grad_norm=0.5, loss=1.26025 +I0916 23:48:04.706995 140109961065664 submission.py:307] 25500) loss = 1.260, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0916 23:54:59.194828 140087195002624 logging_writer.py:48] [26000] global_step=26000, grad_norm=0.5, loss=1.30992 +I0916 23:54:59.201754 140109961065664 submission.py:307] 26000) loss = 1.310, grad_norm = 0.500 +I0917 00:00:06.102139 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 00:00:23.197395 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 00:00:43.140334 140109961065664 spec.py:363] Evaluating on the test split. +I0917 00:00:53.582236 140109961065664 submission_runner.py:516] Time since start: 21050.81s, Step: 26461, {'train/ctc_loss': 0.22989876530732561, 'train/wer': 0.08181480104127929, 'validation/ctc_loss': 0.4952058767204139, 'validation/wer': 0.1402500844880027, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2620092929382541, 'test/wer': 0.08490240285987041, 'test/num_examples': 2472, 'score': 20293.84050965309, 'total_duration': 21050.80857372284, 'accumulated_submission_time': 20293.84050965309, 'accumulated_eval_time': 720.5358953475952, 'accumulated_logging_time': 1.6216716766357422} +I0917 00:00:53.744749 140087195002624 logging_writer.py:48] [26461] accumulated_eval_time=720.536, accumulated_logging_time=1.62167, accumulated_submission_time=20293.8, global_step=26461, preemption_count=0, score=20293.8, test/ctc_loss=0.262009, test/num_examples=2472, test/wer=0.0849024, total_duration=21050.8, train/ctc_loss=0.229899, train/wer=0.0818148, validation/ctc_loss=0.495206, validation/num_examples=5348, validation/wer=0.14025 +I0917 00:01:32.122941 140087186609920 logging_writer.py:48] [26500] global_step=26500, grad_norm=0.5, loss=1.24665 +I0917 00:01:32.126681 140109961065664 submission.py:307] 26500) loss = 1.247, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 00:08:48.779618 140087195002624 logging_writer.py:48] [27000] global_step=27000, grad_norm=0.5, loss=1.27238 +I0917 00:08:48.786599 140109961065664 submission.py:307] 27000) loss = 1.272, grad_norm = 0.500 +I0917 00:14:15.959209 140087186609920 logging_writer.py:48] [27500] global_step=27500, grad_norm=0.5, loss=1.22208 +I0917 00:14:15.963212 140109961065664 submission.py:307] 27500) loss = 1.222, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 00:21:38.830723 140087195002624 logging_writer.py:48] [28000] global_step=28000, grad_norm=0.5, loss=1.20667 +I0917 00:21:38.837832 140109961065664 submission.py:307] 28000) loss = 1.207, grad_norm = 0.500 +I0917 00:25:01.695441 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 00:25:19.200227 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 00:25:39.530007 140109961065664 spec.py:363] Evaluating on the test split. +I0917 00:25:50.035749 140109961065664 submission_runner.py:516] Time since start: 22547.26s, Step: 28356, {'train/ctc_loss': 0.21683521569221959, 'train/wer': 0.07815523253620493, 'validation/ctc_loss': 0.48093368165436007, 'validation/wer': 0.13559600251050066, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2542665807068198, 'test/wer': 0.08171348485771739, 'test/num_examples': 2472, 'score': 21739.44190144539, 'total_duration': 22547.26208972931, 'accumulated_submission_time': 21739.44190144539, 'accumulated_eval_time': 768.8759837150574, 'accumulated_logging_time': 1.7939543724060059} +I0917 00:25:50.097358 140087195002624 logging_writer.py:48] [28356] accumulated_eval_time=768.876, accumulated_logging_time=1.79395, accumulated_submission_time=21739.4, global_step=28356, preemption_count=0, score=21739.4, test/ctc_loss=0.254267, test/num_examples=2472, test/wer=0.0817135, total_duration=22547.3, train/ctc_loss=0.216835, train/wer=0.0781552, validation/ctc_loss=0.480934, validation/num_examples=5348, validation/wer=0.135596 +I0917 00:27:53.556892 140087186609920 logging_writer.py:48] [28500] global_step=28500, grad_norm=0.5, loss=1.17407 +I0917 00:27:53.560758 140109961065664 submission.py:307] 28500) loss = 1.174, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 00:35:35.044143 140087195002624 logging_writer.py:48] [29000] global_step=29000, grad_norm=0.5, loss=1.17926 +I0917 00:35:35.051163 140109961065664 submission.py:307] 29000) loss = 1.179, grad_norm = 0.500 +I0917 00:40:40.570804 140087186609920 logging_writer.py:48] [29500] global_step=29500, grad_norm=0.5, loss=1.28851 +I0917 00:40:40.574606 140109961065664 submission.py:307] 29500) loss = 1.289, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 00:48:24.971217 140087195002624 logging_writer.py:48] [30000] global_step=30000, grad_norm=0.5, loss=1.27297 +I0917 00:48:24.978363 140109961065664 submission.py:307] 30000) loss = 1.273, grad_norm = 0.500 +I0917 00:49:58.298101 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 00:50:13.756460 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 00:50:33.932606 140109961065664 spec.py:363] Evaluating on the test split. +I0917 00:50:44.435421 140109961065664 submission_runner.py:516] Time since start: 24041.66s, Step: 30172, {'train/ctc_loss': 0.20342630375461115, 'train/wer': 0.07387585493233302, 'validation/ctc_loss': 0.4689482753478501, 'validation/wer': 0.13204267851108, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2475520094038325, 'test/wer': 0.07915422582414235, 'test/num_examples': 2472, 'score': 23185.21777319908, 'total_duration': 24041.661779642105, 'accumulated_submission_time': 23185.21777319908, 'accumulated_eval_time': 815.013129234314, 'accumulated_logging_time': 1.8652231693267822} +I0917 00:50:44.496783 140087195002624 logging_writer.py:48] [30172] accumulated_eval_time=815.013, accumulated_logging_time=1.86522, accumulated_submission_time=23185.2, global_step=30172, preemption_count=0, score=23185.2, test/ctc_loss=0.247552, test/num_examples=2472, test/wer=0.0791542, total_duration=24041.7, train/ctc_loss=0.203426, train/wer=0.0738759, validation/ctc_loss=0.468948, validation/num_examples=5348, validation/wer=0.132043 +I0917 00:54:23.084389 140087186609920 logging_writer.py:48] [30500] global_step=30500, grad_norm=0.5, loss=1.2352 +I0917 00:54:23.088011 140109961065664 submission.py:307] 30500) loss = 1.235, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 01:02:25.031562 140087195002624 logging_writer.py:48] [31000] global_step=31000, grad_norm=0.5, loss=1.19261 +I0917 01:02:25.038805 140109961065664 submission.py:307] 31000) loss = 1.193, grad_norm = 0.500 +I0917 01:07:15.937378 140087186609920 logging_writer.py:48] [31500] global_step=31500, grad_norm=0.5, loss=1.20402 +I0917 01:07:15.941016 140109961065664 submission.py:307] 31500) loss = 1.204, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 01:14:52.656457 140109961065664 spec.py:333] Evaluating on the training split. +I0917 01:15:07.703746 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 01:15:27.771047 140109961065664 spec.py:363] Evaluating on the test split. +I0917 01:15:38.257524 140109961065664 submission_runner.py:516] Time since start: 25535.48s, Step: 31942, {'train/ctc_loss': 0.19449660059460644, 'train/wer': 0.07048576864412717, 'validation/ctc_loss': 0.458238463337603, 'validation/wer': 0.130237049196157, 'validation/num_examples': 5348, 'test/ctc_loss': 0.23996749790309052, 'test/wer': 0.07641216257388338, 'test/num_examples': 2472, 'score': 24630.957630872726, 'total_duration': 25535.483829259872, 'accumulated_submission_time': 24630.957630872726, 'accumulated_eval_time': 860.614027261734, 'accumulated_logging_time': 1.936441421508789} +I0917 01:15:38.482528 140087195002624 logging_writer.py:48] [31942] accumulated_eval_time=860.614, accumulated_logging_time=1.93644, accumulated_submission_time=24631, global_step=31942, preemption_count=0, score=24631, test/ctc_loss=0.239967, test/num_examples=2472, test/wer=0.0764122, total_duration=25535.5, train/ctc_loss=0.194497, train/wer=0.0704858, validation/ctc_loss=0.458238, validation/num_examples=5348, validation/wer=0.130237 +I0917 01:16:12.079342 140087186609920 logging_writer.py:48] [32000] global_step=32000, grad_norm=0.5, loss=1.22527 +I0917 01:16:12.082893 140109961065664 submission.py:307] 32000) loss = 1.225, grad_norm = 0.500 +I0917 01:21:05.018368 140087195002624 logging_writer.py:48] [32500] global_step=32500, grad_norm=0.5, loss=1.12512 +I0917 01:21:05.022045 140109961065664 submission.py:307] 32500) loss = 1.125, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 01:29:17.442787 140087195002624 logging_writer.py:48] [33000] global_step=33000, grad_norm=0.5, loss=1.17444 +I0917 01:29:17.449967 140109961065664 submission.py:307] 33000) loss = 1.174, grad_norm = 0.500 +I0917 01:33:55.539505 140087186609920 logging_writer.py:48] [33500] global_step=33500, grad_norm=0.5, loss=1.23881 +I0917 01:33:55.543333 140109961065664 submission.py:307] 33500) loss = 1.239, grad_norm = 0.500 +I0917 01:39:46.794827 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 01:40:02.792612 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 01:40:22.727773 140109961065664 spec.py:363] Evaluating on the test split. +I0917 01:40:33.925788 140109961065664 submission_runner.py:516] Time since start: 27031.15s, Step: 33864, {'train/ctc_loss': 0.18457960651540417, 'train/wer': 0.06688009658242652, 'validation/ctc_loss': 0.4528886589248041, 'validation/wer': 0.12742721962052816, 'validation/num_examples': 5348, 'test/ctc_loss': 0.23561499935479707, 'test/wer': 0.07531533727377979, 'test/num_examples': 2472, 'score': 26076.77673649788, 'total_duration': 27031.151938199997, 'accumulated_submission_time': 26076.77673649788, 'accumulated_eval_time': 907.7446002960205, 'accumulated_logging_time': 2.1716549396514893} +I0917 01:40:33.984266 140087195002624 logging_writer.py:48] [33864] accumulated_eval_time=907.745, accumulated_logging_time=2.17165, accumulated_submission_time=26076.8, global_step=33864, preemption_count=0, score=26076.8, test/ctc_loss=0.235615, test/num_examples=2472, test/wer=0.0753153, total_duration=27031.2, train/ctc_loss=0.18458, train/wer=0.0668801, validation/ctc_loss=0.452889, validation/num_examples=5348, validation/wer=0.127427 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 01:43:04.669377 140087195002624 logging_writer.py:48] [34000] global_step=34000, grad_norm=0.5, loss=1.14307 +I0917 01:43:04.676477 140109961065664 submission.py:307] 34000) loss = 1.143, grad_norm = 0.500 +I0917 01:47:44.519294 140087186609920 logging_writer.py:48] [34500] global_step=34500, grad_norm=0.5, loss=1.14011 +I0917 01:47:44.523266 140109961065664 submission.py:307] 34500) loss = 1.140, grad_norm = 0.500 +I0917 01:55:49.062460 140087195002624 logging_writer.py:48] [35000] global_step=35000, grad_norm=0.5, loss=1.13489 +I0917 01:55:49.066283 140109961065664 submission.py:307] 35000) loss = 1.135, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 02:00:31.149221 140087113082624 logging_writer.py:48] [35500] global_step=35500, grad_norm=0.5, loss=1.18646 +I0917 02:00:31.157665 140109961065664 submission.py:307] 35500) loss = 1.186, grad_norm = 0.500 +I0917 02:04:42.825839 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 02:04:59.324439 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 02:05:19.531159 140109961065664 spec.py:363] Evaluating on the test split. +I0917 02:05:30.067019 140109961065664 submission_runner.py:516] Time since start: 28527.29s, Step: 35799, {'train/ctc_loss': 0.17938031387386932, 'train/wer': 0.06491826604362379, 'validation/ctc_loss': 0.44804351972197104, 'validation/wer': 0.1254381306425916, 'validation/num_examples': 5348, 'test/ctc_loss': 0.23311304862410479, 'test/wer': 0.07389352669957143, 'test/num_examples': 2472, 'score': 27523.128769874573, 'total_duration': 28527.29335784912, 'accumulated_submission_time': 27523.128769874573, 'accumulated_eval_time': 954.985598564148, 'accumulated_logging_time': 2.241617202758789} +I0917 02:05:30.128322 140087195002624 logging_writer.py:48] [35799] accumulated_eval_time=954.986, accumulated_logging_time=2.24162, accumulated_submission_time=27523.1, global_step=35799, preemption_count=0, score=27523.1, test/ctc_loss=0.233113, test/num_examples=2472, test/wer=0.0738935, total_duration=28527.3, train/ctc_loss=0.17938, train/wer=0.0649183, validation/ctc_loss=0.448044, validation/num_examples=5348, validation/wer=0.125438 +I0917 02:09:15.024567 140087186609920 logging_writer.py:48] [36000] global_step=36000, grad_norm=0.5, loss=1.18189 +I0917 02:09:15.028271 140109961065664 submission.py:307] 36000) loss = 1.182, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 02:14:24.669151 140087195002624 logging_writer.py:48] [36500] global_step=36500, grad_norm=0.5, loss=1.14342 +I0917 02:14:24.676054 140109961065664 submission.py:307] 36500) loss = 1.143, grad_norm = 0.500 +I0917 02:21:56.319395 140087186609920 logging_writer.py:48] [37000] global_step=37000, grad_norm=0.5, loss=1.14881 +I0917 02:21:56.323280 140109961065664 submission.py:307] 37000) loss = 1.149, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 02:27:15.595944 140087195002624 logging_writer.py:48] [37500] global_step=37500, grad_norm=0.5, loss=1.20235 +I0917 02:27:15.603093 140109961065664 submission.py:307] 37500) loss = 1.202, grad_norm = 0.500 +I0917 02:29:38.367058 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 02:29:54.584780 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 02:30:14.776410 140109961065664 spec.py:363] Evaluating on the test split. +I0917 02:30:25.462802 140109961065664 submission_runner.py:516] Time since start: 30022.69s, Step: 37709, {'train/ctc_loss': 0.17753223592231374, 'train/wer': 0.06433618445518781, 'validation/ctc_loss': 0.44627454258338356, 'validation/wer': 0.12461739004489934, 'validation/num_examples': 5348, 'test/ctc_loss': 0.23131225200012903, 'test/wer': 0.07326386773099344, 'test/num_examples': 2472, 'score': 28968.81560564041, 'total_duration': 30022.68916273117, 'accumulated_submission_time': 28968.81560564041, 'accumulated_eval_time': 1002.081166267395, 'accumulated_logging_time': 2.312926769256592} +I0917 02:30:25.616207 140087195002624 logging_writer.py:48] [37709] accumulated_eval_time=1002.08, accumulated_logging_time=2.31293, accumulated_submission_time=28968.8, global_step=37709, preemption_count=0, score=28968.8, test/ctc_loss=0.231312, test/num_examples=2472, test/wer=0.0732639, total_duration=30022.7, train/ctc_loss=0.177532, train/wer=0.0643362, validation/ctc_loss=0.446275, validation/num_examples=5348, validation/wer=0.124617 +I0917 02:35:29.471710 140087186609920 logging_writer.py:48] [38000] global_step=38000, grad_norm=0.5, loss=1.13196 +I0917 02:35:29.475798 140109961065664 submission.py:307] 38000) loss = 1.132, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 02:41:09.254230 140087195002624 logging_writer.py:48] [38500] global_step=38500, grad_norm=0.5, loss=1.19337 +I0917 02:41:09.262719 140109961065664 submission.py:307] 38500) loss = 1.193, grad_norm = 0.500 +I0917 02:48:02.175139 140087186609920 logging_writer.py:48] [39000] global_step=39000, grad_norm=0.5, loss=1.19005 +I0917 02:48:02.178977 140109961065664 submission.py:307] 39000) loss = 1.190, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 02:53:57.926081 140087195002624 logging_writer.py:48] [39500] global_step=39500, grad_norm=0.5, loss=1.15635 +I0917 02:53:57.933043 140109961065664 submission.py:307] 39500) loss = 1.156, grad_norm = 0.500 +I0917 02:54:33.407117 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 02:54:50.225723 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 02:55:10.417534 140109961065664 spec.py:363] Evaluating on the test split. +I0917 02:55:20.824186 140109961065664 submission_runner.py:516] Time since start: 31518.05s, Step: 39565, {'train/ctc_loss': 0.17711536396482894, 'train/wer': 0.06422300192410303, 'validation/ctc_loss': 0.446010584234353, 'validation/wer': 0.12444358615362333, 'validation/num_examples': 5348, 'test/ctc_loss': 0.23132034223982192, 'test/wer': 0.07328417931062499, 'test/num_examples': 2472, 'score': 30414.163591861725, 'total_duration': 31518.050532341003, 'accumulated_submission_time': 30414.163591861725, 'accumulated_eval_time': 1049.497998714447, 'accumulated_logging_time': 2.476557493209839} +I0917 02:55:20.878746 140087195002624 logging_writer.py:48] [39565] accumulated_eval_time=1049.5, accumulated_logging_time=2.47656, accumulated_submission_time=30414.2, global_step=39565, preemption_count=0, score=30414.2, test/ctc_loss=0.23132, test/num_examples=2472, test/wer=0.0732842, total_duration=31518.1, train/ctc_loss=0.177115, train/wer=0.064223, validation/ctc_loss=0.446011, validation/num_examples=5348, validation/wer=0.124444 +I0917 03:01:48.832346 140087186609920 logging_writer.py:48] [40000] global_step=40000, grad_norm=0.5, loss=1.25737 +I0917 03:01:48.836893 140109961065664 submission.py:307] 40000) loss = 1.257, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 03:08:02.883858 140087113082624 logging_writer.py:48] [40500] global_step=40500, grad_norm=0.5, loss=1.16537 +I0917 03:08:02.900558 140109961065664 submission.py:307] 40500) loss = 1.165, grad_norm = 0.500 +I0917 03:14:25.148741 140087104689920 logging_writer.py:48] [41000] global_step=41000, grad_norm=0.5, loss=1.10718 +I0917 03:14:25.152720 140109961065664 submission.py:307] 41000) loss = 1.107, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 03:19:28.899509 140109961065664 spec.py:333] Evaluating on the training split. +I0917 03:19:45.435532 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 03:20:05.219906 140109961065664 spec.py:363] Evaluating on the test split. +I0917 03:20:15.636233 140109961065664 submission_runner.py:516] Time since start: 33012.86s, Step: 41333, {'train/ctc_loss': 0.17631941461586403, 'train/wer': 0.06386189575349922, 'validation/ctc_loss': 0.4440256723898684, 'validation/wer': 0.12439530729493554, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2310354196238467, 'test/wer': 0.07314199825320415, 'test/num_examples': 2472, 'score': 31859.80902147293, 'total_duration': 33012.86257123947, 'accumulated_submission_time': 31859.80902147293, 'accumulated_eval_time': 1096.234617948532, 'accumulated_logging_time': 2.5408711433410645} +I0917 03:20:15.689889 140087195002624 logging_writer.py:48] [41333] accumulated_eval_time=1096.23, accumulated_logging_time=2.54087, accumulated_submission_time=31859.8, global_step=41333, preemption_count=0, score=31859.8, test/ctc_loss=0.231035, test/num_examples=2472, test/wer=0.073142, total_duration=33012.9, train/ctc_loss=0.176319, train/wer=0.0638619, validation/ctc_loss=0.444026, validation/num_examples=5348, validation/wer=0.124395 +I0917 03:21:48.303580 140087186609920 logging_writer.py:48] [41500] global_step=41500, grad_norm=0.5, loss=1.23946 +I0917 03:21:48.307091 140109961065664 submission.py:307] 41500) loss = 1.239, grad_norm = 0.500 +I0917 03:28:11.331256 140087195002624 logging_writer.py:48] [42000] global_step=42000, grad_norm=0.5, loss=1.18955 +I0917 03:28:11.334992 140109961065664 submission.py:307] 42000) loss = 1.190, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 03:34:56.505960 140087195002624 logging_writer.py:48] [42500] global_step=42500, grad_norm=0.5, loss=1.1515 +I0917 03:34:56.513172 140109961065664 submission.py:307] 42500) loss = 1.152, grad_norm = 0.500 +I0917 03:40:50.230519 140087186609920 logging_writer.py:48] [43000] global_step=43000, grad_norm=0.5, loss=1.12411 +I0917 03:40:50.234323 140109961065664 submission.py:307] 43000) loss = 1.124, grad_norm = 0.500 +I0917 03:44:23.767216 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 03:44:42.627083 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 03:45:02.475934 140109961065664 spec.py:363] Evaluating on the test split. +I0917 03:45:13.836267 140109961065664 submission_runner.py:516] Time since start: 34511.06s, Step: 43191, {'train/ctc_loss': 0.17633806186517761, 'train/wer': 0.06415293654771721, 'validation/ctc_loss': 0.44561408259242513, 'validation/wer': 0.12432771689277265, 'validation/num_examples': 5348, 'test/ctc_loss': 0.23101117410800698, 'test/wer': 0.07281701297909939, 'test/num_examples': 2472, 'score': 33305.35944676399, 'total_duration': 34511.06227684021, 'accumulated_submission_time': 33305.35944676399, 'accumulated_eval_time': 1146.3031606674194, 'accumulated_logging_time': 2.6046295166015625} +I0917 03:45:14.011802 140087195002624 logging_writer.py:48] [43191] accumulated_eval_time=1146.3, accumulated_logging_time=2.60463, accumulated_submission_time=33305.4, global_step=43191, preemption_count=0, score=33305.4, test/ctc_loss=0.231011, test/num_examples=2472, test/wer=0.072817, total_duration=34511.1, train/ctc_loss=0.176338, train/wer=0.0641529, validation/ctc_loss=0.445614, validation/num_examples=5348, validation/wer=0.124328 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 03:48:42.449762 140087195002624 logging_writer.py:48] [43500] global_step=43500, grad_norm=0.5, loss=1.1478 +I0917 03:48:42.456721 140109961065664 submission.py:307] 43500) loss = 1.148, grad_norm = 0.500 +I0917 03:54:31.688767 140087186609920 logging_writer.py:48] [44000] global_step=44000, grad_norm=0.5, loss=1.21729 +I0917 03:54:31.692675 140109961065664 submission.py:307] 44000) loss = 1.217, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 04:01:40.721857 140087113082624 logging_writer.py:48] [44500] global_step=44500, grad_norm=0.5, loss=1.25674 +I0917 04:01:40.730319 140109961065664 submission.py:307] 44500) loss = 1.257, grad_norm = 0.500 +I0917 04:07:07.745018 140087104689920 logging_writer.py:48] [45000] global_step=45000, grad_norm=0.5, loss=1.1314 +I0917 04:07:07.748782 140109961065664 submission.py:307] 45000) loss = 1.131, grad_norm = 0.500 +I0917 04:09:24.053903 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 04:09:42.298315 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 04:10:02.166182 140109961065664 spec.py:363] Evaluating on the test split. +I0917 04:10:12.593343 140109961065664 submission_runner.py:516] Time since start: 36009.82s, Step: 45127, {'train/ctc_loss': 0.17809340363754275, 'train/wer': 0.06470268027012897, 'validation/ctc_loss': 0.4487201674578059, 'validation/wer': 0.12503258822961424, 'validation/num_examples': 5348, 'test/ctc_loss': 0.23460470231950448, 'test/wer': 0.0750715983182012, 'test/num_examples': 2472, 'score': 34752.82623553276, 'total_duration': 36009.81967043877, 'accumulated_submission_time': 34752.82623553276, 'accumulated_eval_time': 1194.8423812389374, 'accumulated_logging_time': 2.794597625732422} +I0917 04:10:12.663690 140087195002624 logging_writer.py:48] [45127] accumulated_eval_time=1194.84, accumulated_logging_time=2.7946, accumulated_submission_time=34752.8, global_step=45127, preemption_count=0, score=34752.8, test/ctc_loss=0.234605, test/num_examples=2472, test/wer=0.0750716, total_duration=36009.8, train/ctc_loss=0.178093, train/wer=0.0647027, validation/ctc_loss=0.44872, validation/num_examples=5348, validation/wer=0.125033 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 04:15:31.060655 140087195002624 logging_writer.py:48] [45500] global_step=45500, grad_norm=0.5, loss=1.19026 +I0917 04:15:31.067584 140109961065664 submission.py:307] 45500) loss = 1.190, grad_norm = 0.500 +I0917 04:20:50.618110 140087186609920 logging_writer.py:48] [46000] global_step=46000, grad_norm=0.5, loss=1.17092 +I0917 04:20:50.622110 140109961065664 submission.py:307] 46000) loss = 1.171, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 04:28:23.816347 140087195002624 logging_writer.py:48] [46500] global_step=46500, grad_norm=0.5, loss=1.21225 +I0917 04:28:23.823407 140109961065664 submission.py:307] 46500) loss = 1.212, grad_norm = 0.500 +I0917 04:33:29.575380 140087186609920 logging_writer.py:48] [47000] global_step=47000, grad_norm=0.5, loss=1.31844 +I0917 04:33:29.579245 140109961065664 submission.py:307] 47000) loss = 1.318, grad_norm = 0.500 +I0917 04:34:20.838697 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 04:34:37.685787 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 04:34:57.583198 140109961065664 spec.py:363] Evaluating on the test split. +I0917 04:35:08.187111 140109961065664 submission_runner.py:516] Time since start: 37505.41s, Step: 47058, {'train/ctc_loss': 0.1830817994424512, 'train/wer': 0.06686392764941441, 'validation/ctc_loss': 0.4555611084425859, 'validation/wer': 0.12799691015304399, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2380288143589909, 'test/wer': 0.07456380882741251, 'test/num_examples': 2472, 'score': 36198.51755785942, 'total_duration': 37505.41343545914, 'accumulated_submission_time': 36198.51755785942, 'accumulated_eval_time': 1242.190561056137, 'accumulated_logging_time': 2.874882936477661} +I0917 04:35:08.297717 140087195002624 logging_writer.py:48] [47058] accumulated_eval_time=1242.19, accumulated_logging_time=2.87488, accumulated_submission_time=36198.5, global_step=47058, preemption_count=0, score=36198.5, test/ctc_loss=0.238029, test/num_examples=2472, test/wer=0.0745638, total_duration=37505.4, train/ctc_loss=0.183082, train/wer=0.0668639, validation/ctc_loss=0.455561, validation/num_examples=5348, validation/wer=0.127997 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 04:42:11.064790 140087195002624 logging_writer.py:48] [47500] global_step=47500, grad_norm=0.5, loss=1.15122 +I0917 04:42:11.071672 140109961065664 submission.py:307] 47500) loss = 1.151, grad_norm = 0.500 +I0917 04:47:06.411663 140087186609920 logging_writer.py:48] [48000] global_step=48000, grad_norm=0.5, loss=1.23485 +I0917 04:47:06.415665 140109961065664 submission.py:307] 48000) loss = 1.235, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 04:54:56.877376 140087195002624 logging_writer.py:48] [48500] global_step=48500, grad_norm=0.5, loss=1.23766 +I0917 04:54:56.884296 140109961065664 submission.py:307] 48500) loss = 1.238, grad_norm = 0.500 +I0917 04:59:16.278450 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 04:59:33.204295 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 04:59:53.044098 140109961065664 spec.py:363] Evaluating on the test split. +I0917 05:00:03.320361 140109961065664 submission_runner.py:516] Time since start: 39000.55s, Step: 48961, {'train/ctc_loss': 0.19202384637087944, 'train/wer': 0.07039953433472926, 'validation/ctc_loss': 0.472720902871961, 'validation/wer': 0.13161782455462753, 'validation/num_examples': 5348, 'test/ctc_loss': 0.24601834190915542, 'test/wer': 0.0781589584221965, 'test/num_examples': 2472, 'score': 37644.09406256676, 'total_duration': 39000.54671049118, 'accumulated_submission_time': 37644.09406256676, 'accumulated_eval_time': 1289.2322795391083, 'accumulated_logging_time': 2.995460033416748} +I0917 05:00:03.515017 140087195002624 logging_writer.py:48] [48961] accumulated_eval_time=1289.23, accumulated_logging_time=2.99546, accumulated_submission_time=37644.1, global_step=48961, preemption_count=0, score=37644.1, test/ctc_loss=0.246018, test/num_examples=2472, test/wer=0.078159, total_duration=39000.5, train/ctc_loss=0.192024, train/wer=0.0703995, validation/ctc_loss=0.472721, validation/num_examples=5348, validation/wer=0.131618 +I0917 05:00:34.002535 140087186609920 logging_writer.py:48] [49000] global_step=49000, grad_norm=0.5, loss=1.24168 +I0917 05:00:34.005917 140109961065664 submission.py:307] 49000) loss = 1.242, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 05:08:48.764239 140087113082624 logging_writer.py:48] [49500] global_step=49500, grad_norm=0.5, loss=1.19301 +I0917 05:08:48.772869 140109961065664 submission.py:307] 49500) loss = 1.193, grad_norm = 0.500 +I0917 05:13:32.082642 140087104689920 logging_writer.py:48] [50000] global_step=50000, grad_norm=0.5, loss=1.23516 +I0917 05:13:32.086461 140109961065664 submission.py:307] 50000) loss = 1.235, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 05:21:38.073406 140087195002624 logging_writer.py:48] [50500] global_step=50500, grad_norm=0.5, loss=1.17981 +I0917 05:21:38.080475 140109961065664 submission.py:307] 50500) loss = 1.180, grad_norm = 0.500 +I0917 05:24:11.467090 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 05:24:26.638488 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 05:24:46.590098 140109961065664 spec.py:363] Evaluating on the test split. +I0917 05:24:56.977771 140109961065664 submission_runner.py:516] Time since start: 40494.20s, Step: 50783, {'train/ctc_loss': 0.20528400142334968, 'train/wer': 0.07593469906920843, 'validation/ctc_loss': 0.4910192353576452, 'validation/wer': 0.13853135711871772, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2585415288889606, 'test/wer': 0.08281031015782098, 'test/num_examples': 2472, 'score': 39089.643002033234, 'total_duration': 40494.20410013199, 'accumulated_submission_time': 39089.643002033234, 'accumulated_eval_time': 1334.7426974773407, 'accumulated_logging_time': 3.2001163959503174} +I0917 05:24:57.037075 140087195002624 logging_writer.py:48] [50783] accumulated_eval_time=1334.74, accumulated_logging_time=3.20012, accumulated_submission_time=39089.6, global_step=50783, preemption_count=0, score=39089.6, test/ctc_loss=0.258542, test/num_examples=2472, test/wer=0.0828103, total_duration=40494.2, train/ctc_loss=0.205284, train/wer=0.0759347, validation/ctc_loss=0.491019, validation/num_examples=5348, validation/wer=0.138531 +I0917 05:27:10.934328 140087186609920 logging_writer.py:48] [51000] global_step=51000, grad_norm=0.5, loss=1.26482 +I0917 05:27:10.938003 140109961065664 submission.py:307] 51000) loss = 1.265, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 05:35:37.395109 140087195002624 logging_writer.py:48] [51500] global_step=51500, grad_norm=0.5, loss=1.25562 +I0917 05:35:37.402726 140109961065664 submission.py:307] 51500) loss = 1.256, grad_norm = 0.500 +I0917 05:40:11.184337 140087186609920 logging_writer.py:48] [52000] global_step=52000, grad_norm=0.5, loss=1.23245 +I0917 05:40:11.188227 140109961065664 submission.py:307] 52000) loss = 1.232, grad_norm = 0.500 +I0917 05:48:14.689477 140087195002624 logging_writer.py:48] [52500] global_step=52500, grad_norm=0.5, loss=1.31018 +I0917 05:48:14.693280 140109961065664 submission.py:307] 52500) loss = 1.310, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 05:49:05.325536 140109961065664 spec.py:333] Evaluating on the training split. +I0917 05:49:20.524363 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 05:49:40.444987 140109961065664 spec.py:363] Evaluating on the test split. +I0917 05:49:50.974487 140109961065664 submission_runner.py:516] Time since start: 41988.20s, Step: 52559, {'train/ctc_loss': 0.21611553977799114, 'train/wer': 0.07919543389331737, 'validation/ctc_loss': 0.49940767569883965, 'validation/wer': 0.14022111717279004, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2638593114878379, 'test/wer': 0.08348059228566206, 'test/num_examples': 2472, 'score': 40535.483557224274, 'total_duration': 41988.20076799393, 'accumulated_submission_time': 40535.483557224274, 'accumulated_eval_time': 1380.3914070129395, 'accumulated_logging_time': 3.269481897354126} +I0917 05:49:51.026737 140087195002624 logging_writer.py:48] [52559] accumulated_eval_time=1380.39, accumulated_logging_time=3.26948, accumulated_submission_time=40535.5, global_step=52559, preemption_count=0, score=40535.5, test/ctc_loss=0.263859, test/num_examples=2472, test/wer=0.0834806, total_duration=41988.2, train/ctc_loss=0.216116, train/wer=0.0791954, validation/ctc_loss=0.499408, validation/num_examples=5348, validation/wer=0.140221 +I0917 05:53:56.754701 140087186609920 logging_writer.py:48] [53000] global_step=53000, grad_norm=0.5, loss=1.42109 +I0917 05:53:56.758505 140109961065664 submission.py:307] 53000) loss = 1.421, grad_norm = 0.500 +I0917 06:01:48.264329 140087195002624 logging_writer.py:48] [53500] global_step=53500, grad_norm=0.5, loss=1.30733 +I0917 06:01:48.268144 140109961065664 submission.py:307] 53500) loss = 1.307, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 06:06:55.456576 140087195002624 logging_writer.py:48] [54000] global_step=54000, grad_norm=0.5, loss=1.2473 +I0917 06:06:55.463955 140109961065664 submission.py:307] 54000) loss = 1.247, grad_norm = 0.500 +I0917 06:14:01.563663 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 06:14:17.004713 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 06:14:36.942864 140109961065664 spec.py:363] Evaluating on the test split. +I0917 06:14:47.547814 140109961065664 submission_runner.py:516] Time since start: 43484.77s, Step: 54474, {'train/ctc_loss': 0.23243013269156265, 'train/wer': 0.08429403743646957, 'validation/ctc_loss': 0.5142982916729455, 'validation/wer': 0.1447400183459663, 'validation/num_examples': 5348, 'test/ctc_loss': 0.27086775255661655, 'test/wer': 0.0870960534600776, 'test/num_examples': 2472, 'score': 41983.547627449036, 'total_duration': 43484.77414178848, 'accumulated_submission_time': 41983.547627449036, 'accumulated_eval_time': 1426.375376701355, 'accumulated_logging_time': 3.331726312637329} +I0917 06:14:47.776369 140087195002624 logging_writer.py:48] [54474] accumulated_eval_time=1426.38, accumulated_logging_time=3.33173, accumulated_submission_time=41983.5, global_step=54474, preemption_count=0, score=41983.5, test/ctc_loss=0.270868, test/num_examples=2472, test/wer=0.0870961, total_duration=43484.8, train/ctc_loss=0.23243, train/wer=0.084294, validation/ctc_loss=0.514298, validation/num_examples=5348, validation/wer=0.14474 +I0917 06:15:14.615291 140087186609920 logging_writer.py:48] [54500] global_step=54500, grad_norm=0.5, loss=1.3073 +I0917 06:15:14.618904 140109961065664 submission.py:307] 54500) loss = 1.307, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 06:20:43.972286 140087195002624 logging_writer.py:48] [55000] global_step=55000, grad_norm=0.5, loss=1.26253 +I0917 06:20:43.979401 140109961065664 submission.py:307] 55000) loss = 1.263, grad_norm = 0.500 +I0917 06:27:55.778035 140087186609920 logging_writer.py:48] [55500] global_step=55500, grad_norm=0.5, loss=1.35444 +I0917 06:27:55.782031 140109961065664 submission.py:307] 55500) loss = 1.354, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 06:33:38.541101 140087195002624 logging_writer.py:48] [56000] global_step=56000, grad_norm=0.5, loss=1.23846 +I0917 06:33:38.548239 140109961065664 submission.py:307] 56000) loss = 1.238, grad_norm = 0.500 +I0917 06:38:55.380685 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 06:39:11.528729 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 06:39:31.576321 140109961065664 spec.py:363] Evaluating on the test split. +I0917 06:39:42.299528 140109961065664 submission_runner.py:516] Time since start: 44979.53s, Step: 56411, {'train/ctc_loss': 0.25572556091768156, 'train/wer': 0.09282584442252656, 'validation/ctc_loss': 0.5407247361299478, 'validation/wer': 0.15061072756240043, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2903184782082715, 'test/wer': 0.09408323685333009, 'test/num_examples': 2472, 'score': 43428.78943896294, 'total_duration': 44979.52585673332, 'accumulated_submission_time': 43428.78943896294, 'accumulated_eval_time': 1473.2940378189087, 'accumulated_logging_time': 3.570307731628418} +I0917 06:39:42.374689 140087195002624 logging_writer.py:48] [56411] accumulated_eval_time=1473.29, accumulated_logging_time=3.57031, accumulated_submission_time=43428.8, global_step=56411, preemption_count=0, score=43428.8, test/ctc_loss=0.290318, test/num_examples=2472, test/wer=0.0940832, total_duration=44979.5, train/ctc_loss=0.255726, train/wer=0.0928258, validation/ctc_loss=0.540725, validation/num_examples=5348, validation/wer=0.150611 +I0917 06:41:20.096638 140087186609920 logging_writer.py:48] [56500] global_step=56500, grad_norm=0.5, loss=1.34657 +I0917 06:41:20.100218 140109961065664 submission.py:307] 56500) loss = 1.347, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 06:47:21.968769 140087195002624 logging_writer.py:48] [57000] global_step=57000, grad_norm=0.5, loss=1.27764 +I0917 06:47:21.975896 140109961065664 submission.py:307] 57000) loss = 1.278, grad_norm = 0.500 +I0917 06:53:54.458807 140087186609920 logging_writer.py:48] [57500] global_step=57500, grad_norm=0.5, loss=1.43779 +I0917 06:53:54.462640 140109961065664 submission.py:307] 57500) loss = 1.438, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 07:00:18.305011 140087195002624 logging_writer.py:48] [58000] global_step=58000, grad_norm=0.5, loss=1.3499 +I0917 07:00:18.312382 140109961065664 submission.py:307] 58000) loss = 1.350, grad_norm = 0.500 +I0917 07:03:50.187782 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 07:04:05.924451 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 07:04:25.970553 140109961065664 spec.py:363] Evaluating on the test split. +I0917 07:04:36.706556 140109961065664 submission_runner.py:516] Time since start: 46473.93s, Step: 58326, {'train/ctc_loss': 0.26346462997119613, 'train/wer': 0.09424871052759229, 'validation/ctc_loss': 0.5477415839423849, 'validation/wer': 0.15239704533384832, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2869752937689528, 'test/wer': 0.09400199053480389, 'test/num_examples': 2472, 'score': 44874.081411361694, 'total_duration': 46473.932918787, 'accumulated_submission_time': 44874.081411361694, 'accumulated_eval_time': 1519.8126411437988, 'accumulated_logging_time': 3.6556499004364014} +I0917 07:04:36.791791 140087195002624 logging_writer.py:48] [58326] accumulated_eval_time=1519.81, accumulated_logging_time=3.65565, accumulated_submission_time=44874.1, global_step=58326, preemption_count=0, score=44874.1, test/ctc_loss=0.286975, test/num_examples=2472, test/wer=0.094002, total_duration=46473.9, train/ctc_loss=0.263465, train/wer=0.0942487, validation/ctc_loss=0.547742, validation/num_examples=5348, validation/wer=0.152397 +I0917 07:07:32.410651 140087186609920 logging_writer.py:48] [58500] global_step=58500, grad_norm=0.5, loss=1.35143 +I0917 07:07:32.414596 140109961065664 submission.py:307] 58500) loss = 1.351, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 07:14:09.622814 140087195002624 logging_writer.py:48] [59000] global_step=59000, grad_norm=0.5, loss=1.34242 +I0917 07:14:09.629848 140109961065664 submission.py:307] 59000) loss = 1.342, grad_norm = 0.500 +I0917 07:20:07.944331 140087186609920 logging_writer.py:48] [59500] global_step=59500, grad_norm=0.5, loss=1.40685 +I0917 07:20:07.948222 140109961065664 submission.py:307] 59500) loss = 1.407, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 07:27:01.806437 140087195002624 logging_writer.py:48] [60000] global_step=60000, grad_norm=0.5, loss=1.35449 +I0917 07:27:01.813356 140109961065664 submission.py:307] 60000) loss = 1.354, grad_norm = 0.500 +I0917 07:28:44.631356 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 07:29:00.300144 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 07:29:20.198191 140109961065664 spec.py:363] Evaluating on the test split. +I0917 07:29:30.641041 140109961065664 submission_runner.py:516] Time since start: 47967.87s, Step: 60188, {'train/ctc_loss': 0.26306591835531523, 'train/wer': 0.09519728793096943, 'validation/ctc_loss': 0.535008598145218, 'validation/wer': 0.15052382561676242, 'validation/num_examples': 5348, 'test/ctc_loss': 0.2884309068326989, 'test/wer': 0.09318952734954197, 'test/num_examples': 2472, 'score': 46319.49430727959, 'total_duration': 47967.867362737656, 'accumulated_submission_time': 46319.49430727959, 'accumulated_eval_time': 1565.8220901489258, 'accumulated_logging_time': 3.750631093978882} +I0917 07:29:30.706529 140087195002624 logging_writer.py:48] [60188] accumulated_eval_time=1565.82, accumulated_logging_time=3.75063, accumulated_submission_time=46319.5, global_step=60188, preemption_count=0, score=46319.5, test/ctc_loss=0.288431, test/num_examples=2472, test/wer=0.0931895, total_duration=47967.9, train/ctc_loss=0.263066, train/wer=0.0951973, validation/ctc_loss=0.535009, validation/num_examples=5348, validation/wer=0.150524 +I0917 07:33:52.601775 140087186609920 logging_writer.py:48] [60500] global_step=60500, grad_norm=0.5, loss=1.46788 +I0917 07:33:52.605588 140109961065664 submission.py:307] 60500) loss = 1.468, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 07:40:59.301207 140087113082624 logging_writer.py:48] [61000] global_step=61000, grad_norm=0.5, loss=1.3515 +I0917 07:40:59.309723 140109961065664 submission.py:307] 61000) loss = 1.352, grad_norm = 0.500 +I0917 07:46:29.409604 140087104689920 logging_writer.py:48] [61500] global_step=61500, grad_norm=0.5, loss=1.31355 +I0917 07:46:29.413610 140109961065664 submission.py:307] 61500) loss = 1.314, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 07:53:38.477574 140109961065664 spec.py:333] Evaluating on the training split. +I0917 07:53:54.259930 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 07:54:14.036742 140109961065664 spec.py:363] Evaluating on the test split. +I0917 07:54:24.609246 140109961065664 submission_runner.py:516] Time since start: 49461.84s, Step: 61973, {'train/ctc_loss': 0.2825832322670844, 'train/wer': 0.0995575101999019, 'validation/ctc_loss': 0.5602312139749096, 'validation/wer': 0.15486892289866266, 'validation/num_examples': 5348, 'test/ctc_loss': 0.30500430471320733, 'test/wer': 0.0973534011740093, 'test/num_examples': 2472, 'score': 47764.89419078827, 'total_duration': 49461.83552980423, 'accumulated_submission_time': 47764.89419078827, 'accumulated_eval_time': 1611.953557252884, 'accumulated_logging_time': 3.8260538578033447} +I0917 07:54:24.814629 140087195002624 logging_writer.py:48] [61973] accumulated_eval_time=1611.95, accumulated_logging_time=3.82605, accumulated_submission_time=47764.9, global_step=61973, preemption_count=0, score=47764.9, test/ctc_loss=0.305004, test/num_examples=2472, test/wer=0.0973534, total_duration=49461.8, train/ctc_loss=0.282583, train/wer=0.0995575, validation/ctc_loss=0.560231, validation/num_examples=5348, validation/wer=0.154869 +I0917 07:54:41.002780 140087186609920 logging_writer.py:48] [62000] global_step=62000, grad_norm=0.5, loss=1.36407 +I0917 07:54:41.006200 140109961065664 submission.py:307] 62000) loss = 1.364, grad_norm = 0.500 +I0917 08:00:14.540819 140087195002624 logging_writer.py:48] [62500] global_step=62500, grad_norm=0.5, loss=1.26974 +I0917 08:00:14.544714 140109961065664 submission.py:307] 62500) loss = 1.270, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 08:07:48.753083 140087195002624 logging_writer.py:48] [63000] global_step=63000, grad_norm=0.5, loss=1.2949 +I0917 08:07:48.760031 140109961065664 submission.py:307] 63000) loss = 1.295, grad_norm = 0.500 +I0917 08:12:58.788749 140087186609920 logging_writer.py:48] [63500] global_step=63500, grad_norm=0.5, loss=1.3678 +I0917 08:12:58.792534 140109961065664 submission.py:307] 63500) loss = 1.368, grad_norm = 0.500 +I0917 08:18:33.397385 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 08:18:48.697776 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 08:19:08.868012 140109961065664 spec.py:363] Evaluating on the test split. +I0917 08:19:19.426285 140109961065664 submission_runner.py:516] Time since start: 50956.65s, Step: 63809, {'train/ctc_loss': 0.28190040342277695, 'train/wer': 0.10009108498930155, 'validation/ctc_loss': 0.5507110049979907, 'validation/wer': 0.15553517114855406, 'validation/num_examples': 5348, 'test/ctc_loss': 0.3006939207771469, 'test/wer': 0.09775963276664026, 'test/num_examples': 2472, 'score': 49210.938774585724, 'total_duration': 50956.652635097504, 'accumulated_submission_time': 49210.938774585724, 'accumulated_eval_time': 1657.9822750091553, 'accumulated_logging_time': 4.041477918624878} +I0917 08:19:19.493537 140087195002624 logging_writer.py:48] [63809] accumulated_eval_time=1657.98, accumulated_logging_time=4.04148, accumulated_submission_time=49210.9, global_step=63809, preemption_count=0, score=49210.9, test/ctc_loss=0.300694, test/num_examples=2472, test/wer=0.0977596, total_duration=50956.7, train/ctc_loss=0.2819, train/wer=0.100091, validation/ctc_loss=0.550711, validation/num_examples=5348, validation/wer=0.155535 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 08:21:33.807965 140087195002624 logging_writer.py:48] [64000] global_step=64000, grad_norm=0.5, loss=1.38026 +I0917 08:21:33.814961 140109961065664 submission.py:307] 64000) loss = 1.380, grad_norm = 0.500 +I0917 08:26:41.109752 140087186609920 logging_writer.py:48] [64500] global_step=64500, grad_norm=0.5, loss=1.38657 +I0917 08:26:41.113654 140109961065664 submission.py:307] 64500) loss = 1.387, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 08:34:30.125860 140087195002624 logging_writer.py:48] [65000] global_step=65000, grad_norm=0.5, loss=1.36659 +I0917 08:34:30.132835 140109961065664 submission.py:307] 65000) loss = 1.367, grad_norm = 0.500 +I0917 08:39:26.040396 140087186609920 logging_writer.py:48] [65500] global_step=65500, grad_norm=0.5, loss=1.3203 +I0917 08:39:26.044239 140109961065664 submission.py:307] 65500) loss = 1.320, grad_norm = 0.500 +I0917 08:43:28.263851 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 08:43:44.109250 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 08:44:04.002876 140109961065664 spec.py:363] Evaluating on the test split. +I0917 08:44:14.504805 140109961065664 submission_runner.py:516] Time since start: 52451.73s, Step: 65748, {'train/ctc_loss': 0.29636972139404044, 'train/wer': 0.10388539460281015, 'validation/ctc_loss': 0.568032922144992, 'validation/wer': 0.16094240332158546, 'validation/num_examples': 5348, 'test/ctc_loss': 0.30728605474546744, 'test/wer': 0.09729246643511466, 'test/num_examples': 2472, 'score': 50657.33596086502, 'total_duration': 52451.73114442825, 'accumulated_submission_time': 50657.33596086502, 'accumulated_eval_time': 1704.2230117321014, 'accumulated_logging_time': 4.118767023086548} +I0917 08:44:14.556368 140087195002624 logging_writer.py:48] [65748] accumulated_eval_time=1704.22, accumulated_logging_time=4.11877, accumulated_submission_time=50657.3, global_step=65748, preemption_count=0, score=50657.3, test/ctc_loss=0.307286, test/num_examples=2472, test/wer=0.0972925, total_duration=52451.7, train/ctc_loss=0.29637, train/wer=0.103885, validation/ctc_loss=0.568033, validation/num_examples=5348, validation/wer=0.160942 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 08:48:14.363427 140087195002624 logging_writer.py:48] [66000] global_step=66000, grad_norm=0.5, loss=1.32185 +I0917 08:48:14.370385 140109961065664 submission.py:307] 66000) loss = 1.322, grad_norm = 0.500 +I0917 08:53:07.082302 140087186609920 logging_writer.py:48] [66500] global_step=66500, grad_norm=0.5, loss=1.43195 +I0917 08:53:07.086178 140109961065664 submission.py:307] 66500) loss = 1.432, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 09:01:08.479026 140087195002624 logging_writer.py:48] [67000] global_step=67000, grad_norm=0.5, loss=1.36342 +I0917 09:01:08.486221 140109961065664 submission.py:307] 67000) loss = 1.363, grad_norm = 0.500 +I0917 09:05:47.659230 140087186609920 logging_writer.py:48] [67500] global_step=67500, grad_norm=0.5, loss=1.36398 +I0917 09:05:47.663017 140109961065664 submission.py:307] 67500) loss = 1.364, grad_norm = 0.500 +I0917 09:08:22.719191 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 09:08:39.535580 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 09:08:59.777420 140109961065664 spec.py:363] Evaluating on the test split. +I0917 09:09:10.142562 140109961065664 submission_runner.py:516] Time since start: 53947.37s, Step: 67681, {'train/ctc_loss': 0.3175066219448515, 'train/wer': 0.11081108757633083, 'validation/ctc_loss': 0.5771099793424754, 'validation/wer': 0.1646309081253319, 'validation/num_examples': 5348, 'test/ctc_loss': 0.31436808927188853, 'test/wer': 0.1023500497633701, 'test/num_examples': 2472, 'score': 52103.002204179764, 'total_duration': 53947.36892056465, 'accumulated_submission_time': 52103.002204179764, 'accumulated_eval_time': 1751.6461789608002, 'accumulated_logging_time': 4.180283546447754} +I0917 09:09:10.327973 140087195002624 logging_writer.py:48] [67681] accumulated_eval_time=1751.65, accumulated_logging_time=4.18028, accumulated_submission_time=52103, global_step=67681, preemption_count=0, score=52103, test/ctc_loss=0.314368, test/num_examples=2472, test/wer=0.10235, total_duration=53947.4, train/ctc_loss=0.317507, train/wer=0.110811, validation/ctc_loss=0.57711, validation/num_examples=5348, validation/wer=0.164631 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 09:14:55.784584 140087195002624 logging_writer.py:48] [68000] global_step=68000, grad_norm=0.5, loss=1.35765 +I0917 09:14:55.792330 140109961065664 submission.py:307] 68000) loss = 1.358, grad_norm = 0.500 +I0917 09:19:36.359497 140087186609920 logging_writer.py:48] [68500] global_step=68500, grad_norm=0.5, loss=1.27842 +I0917 09:19:36.363913 140109961065664 submission.py:307] 68500) loss = 1.278, grad_norm = 0.500 +I0917 09:27:37.772784 140087195002624 logging_writer.py:48] [69000] global_step=69000, grad_norm=0.5, loss=1.36234 +I0917 09:27:37.776698 140109961065664 submission.py:307] 69000) loss = 1.362, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 09:32:20.190827 140087195002624 logging_writer.py:48] [69500] global_step=69500, grad_norm=0.5, loss=1.38323 +I0917 09:32:20.197958 140109961065664 submission.py:307] 69500) loss = 1.383, grad_norm = 0.500 +I0917 09:33:18.487940 140109961065664 spec.py:333] Evaluating on the training split. +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 09:33:34.333158 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 09:33:54.075917 140109961065664 spec.py:363] Evaluating on the test split. +I0917 09:34:04.327522 140109961065664 submission_runner.py:516] Time since start: 55441.55s, Step: 69583, {'train/ctc_loss': 0.3075975499856823, 'train/wer': 0.10689820578740009, 'validation/ctc_loss': 0.5708020566480811, 'validation/wer': 0.16114517452807417, 'validation/num_examples': 5348, 'test/ctc_loss': 0.3124170813439577, 'test/wer': 0.09954705177421648, 'test/num_examples': 2472, 'score': 53548.67119860649, 'total_duration': 55441.55384373665, 'accumulated_submission_time': 53548.67119860649, 'accumulated_eval_time': 1797.4855284690857, 'accumulated_logging_time': 4.3754332065582275} +I0917 09:34:04.398793 140087195002624 logging_writer.py:48] [69583] accumulated_eval_time=1797.49, accumulated_logging_time=4.37543, accumulated_submission_time=53548.7, global_step=69583, preemption_count=0, score=53548.7, test/ctc_loss=0.312417, test/num_examples=2472, test/wer=0.0995471, total_duration=55441.6, train/ctc_loss=0.307598, train/wer=0.106898, validation/ctc_loss=0.570802, validation/num_examples=5348, validation/wer=0.161145 +I0917 09:41:13.084788 140087186609920 logging_writer.py:48] [70000] global_step=70000, grad_norm=0.5, loss=1.51756 +I0917 09:41:13.088633 140109961065664 submission.py:307] 70000) loss = 1.518, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 09:46:10.723540 140087195002624 logging_writer.py:48] [70500] global_step=70500, grad_norm=0.5, loss=1.47477 +I0917 09:46:10.730435 140109961065664 submission.py:307] 70500) loss = 1.475, grad_norm = 0.500 +I0917 09:53:43.478559 140087186609920 logging_writer.py:48] [71000] global_step=71000, grad_norm=0.5, loss=1.42896 +I0917 09:53:43.482620 140109961065664 submission.py:307] 71000) loss = 1.429, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 09:58:12.508666 140109961065664 spec.py:333] Evaluating on the training split. +I0917 09:58:27.958988 140109961065664 spec.py:346] Evaluating on the validation split. +I0917 09:58:48.585322 140109961065664 spec.py:363] Evaluating on the test split. +I0917 09:58:59.127866 140109961065664 submission_runner.py:516] Time since start: 56936.35s, Step: 71419, {'train/ctc_loss': 0.311029117607425, 'train/wer': 0.10914029783174609, 'validation/ctc_loss': 0.579972463174352, 'validation/wer': 0.16227489982136822, 'validation/num_examples': 5348, 'test/ctc_loss': 0.313425916591393, 'test/wer': 0.0993845591371641, 'test/num_examples': 2472, 'score': 54994.37272357941, 'total_duration': 56936.354231119156, 'accumulated_submission_time': 54994.37272357941, 'accumulated_eval_time': 1844.1046738624573, 'accumulated_logging_time': 4.456562280654907} +I0917 09:58:59.182763 140087195002624 logging_writer.py:48] [71419] accumulated_eval_time=1844.1, accumulated_logging_time=4.45656, accumulated_submission_time=54994.4, global_step=71419, preemption_count=0, score=54994.4, test/ctc_loss=0.313426, test/num_examples=2472, test/wer=0.0993846, total_duration=56936.4, train/ctc_loss=0.311029, train/wer=0.10914, validation/ctc_loss=0.579972, validation/num_examples=5348, validation/wer=0.162275 +I0917 09:59:46.489027 140087186609920 logging_writer.py:48] [71500] global_step=71500, grad_norm=0.5, loss=1.41722 +I0917 09:59:46.492449 140109961065664 submission.py:307] 71500) loss = 1.417, grad_norm = 0.500 +I0917 10:07:23.259276 140087195002624 logging_writer.py:48] [72000] global_step=72000, grad_norm=0.5, loss=1.45371 +I0917 10:07:23.263124 140109961065664 submission.py:307] 72000) loss = 1.454, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 10:12:57.370534 140087195002624 logging_writer.py:48] [72500] global_step=72500, grad_norm=0.5, loss=1.34369 +I0917 10:12:57.377456 140109961065664 submission.py:307] 72500) loss = 1.344, grad_norm = 0.500 +I0917 10:20:00.174382 140087186609920 logging_writer.py:48] [73000] global_step=73000, grad_norm=0.5, loss=1.45069 +I0917 10:20:00.178271 140109961065664 submission.py:307] 73000) loss = 1.451, grad_norm = 0.500 +/usr/local/lib/python3.11/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock. + self.pid = os.fork() +I0917 10:23:06.155323 140087195002624 logging_writer.py:48] [73196] global_step=73196, preemption_count=0, score=56439.8 +I0917 10:23:07.083648 140109961065664 submission_runner.py:857] Final librispeech_deepspeech score: 56439.75518965721 diff --git a/logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..75ed4a8f --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,40 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/ctc_loss,test/num_examples,test/wer,total_duration,train/ctc_loss,train/wer,validation/ctc_loss,validation/num_examples,validation/wer +67.00936770439148,0.0,54.36974382400513,1,0,54.36974382400513,31.595735208723145,2472,3.6465785144110656,121.9671106338501,32.54331867872724,3.611568332605732,31.47319858850713,5348,3.283097571573408 +110.08822107315063,0.1076393127441406,1499.6107699871063,1991,0,1499.6107699871063,5.847129653526034,2472,0.8991123839701014,1612.9545664787292,6.037100156652686,0.9414253453414608,5.9608374303044,5348,0.8964901269733984 +158.6353361606598,0.3192300796508789,2945.160037517548,3933,0,2945.160037517548,1.2938803511516872,2472,0.349074807547783,3109.749101638794,1.6419855517374973,0.4234104591438011,1.819571591822383,5348,0.4385844638632742 +208.74199318885803,0.4080448150634765,4390.693488121033,5867,0,4390.693488121033,0.512976744467385,2472,0.1638128897284342,4607.824376344681,0.5884155970488655,0.1902005486657935,0.8283974446516476,5348,0.234934582146478 +256.05564308166504,0.4997012615203857,5836.115516662598,7769,0,5836.115516662598,0.4462445157752113,2472,0.1446184469766213,6103.235170602799,0.4873721933060454,0.1617162783427921,0.7424400288200723,5348,0.2120600589002076 +301.57932806015015,0.6819305419921875,7281.850630283356,9590,0,7281.850630283356,0.3939028324408026,2472,0.128998842239961,7597.046008348465,0.4252956714167804,0.1424644687697059,0.6756669056566456,5348,0.1934147636749867 +346.61975502967834,0.7487177848815918,8727.689821004868,11351,0,8727.689821004868,0.3634641247015936,2472,0.1171775028944001,9090.325487852097,0.3833944731500665,0.1296748427571264,0.6379087945172794,5348,0.1844542075025346 +392.4531898498535,0.8087592124938965,10173.496448755264,13269,0,10173.496448755264,0.3382709166720433,2472,0.111023094266041,10584.515745162964,0.3551276908888776,0.1223880436130019,0.6118201343366988,5348,0.1745763530150147 +440.9172077178955,1.01652193069458,11619.135604143145,15193,0,11619.135604143145,0.3243494287937931,2472,0.1052952288099445,12081.224436283112,0.3283039978018091,0.1129184385122425,0.5931179421432339,5348,0.1685414956790421 +488.4095978736877,1.1076455116271973,13065.391492843628,17119,0,13065.391492843628,0.3067811582602103,2472,0.0993033128186379,13577.54152393341,0.3036698365674533,0.1057609908322149,0.5600870449379647,5348,0.1598416453435041 +534.4881474971771,1.189727783203125,14511.024166345596,18990,0,14511.024166345596,0.2995533229966449,2472,0.0980643064611134,15071.766645908356,0.2967907781258949,0.1031631822616025,0.5587584725675607,5348,0.1591657413218751 +580.2749526500702,1.2638778686523438,15956.933192014694,20770,0,15956.933192014694,0.2936532447657913,2472,0.0955456705868015,16565.886620759964,0.2835062776879411,0.0986736085285732,0.5481313652175005,5348,0.1545985612900111 +627.3567457199097,1.4887628555297852,17403.099194288254,22608,0,17403.099194288254,0.2798856982708562,2472,0.0906099567363353,18061.93473124504,0.2634278618592821,0.0929120787319244,0.5222968526314798,5348,0.148640950127939 +673.05593085289,1.5548083782196045,18848.515809059143,24531,0,18848.515809059143,0.2715927237241112,2472,0.0873397924156561,19555.506080389023,0.2469450620715212,0.0870697042702152,0.5107892791214587,5348,0.1445758702264278 +720.5358953475952,1.6216716766357422,20293.84050965309,26461,0,20293.84050965309,0.2620092929382541,2472,0.0849024028598704,21050.80857372284,0.2298987653073256,0.0818148010412792,0.4952058767204139,5348,0.1402500844880027 +768.8759837150574,1.793954372406006,21739.44190144539,28356,0,21739.44190144539,0.2542665807068198,2472,0.0817134848577173,22547.26208972931,0.2168352156922195,0.0781552325362049,0.48093368165436,5348,0.1355960025105006 +815.013129234314,1.8652231693267824,23185.21777319908,30172,0,23185.21777319908,0.2475520094038325,2472,0.0791542258241423,24041.661779642105,0.2034263037546111,0.073875854932333,0.4689482753478501,5348,0.13204267851108 +860.614027261734,1.936441421508789,24630.957630872726,31942,0,24630.957630872726,0.2399674979030905,2472,0.0764121625738833,25535.483829259872,0.1944966005946064,0.0704857686441271,0.458238463337603,5348,0.130237049196157 +907.7446002960204,2.1716549396514893,26076.77673649788,33864,0,26076.77673649788,0.235614999354797,2472,0.0753153372737797,27031.1519382,0.1845796065154041,0.0668800965824265,0.4528886589248041,5348,0.1274272196205281 +954.985598564148,2.241617202758789,27523.128769874573,35799,0,27523.128769874573,0.2331130486241047,2472,0.0738935266995714,28527.29335784912,0.1793803138738693,0.0649182660436237,0.448043519721971,5348,0.1254381306425916 +1002.081166267395,2.312926769256592,28968.81560564041,37709,0,28968.81560564041,0.231312252000129,2472,0.0732638677309934,30022.68916273117,0.1775322359223137,0.0643361844551878,0.4462745425833835,5348,0.1246173900448993 +1049.497998714447,2.476557493209839,30414.163591861725,39565,0,30414.163591861725,0.2313203422398219,2472,0.0732841793106249,31518.050532341003,0.1771153639648289,0.064223001924103,0.446010584234353,5348,0.1244435861536233 +1096.234617948532,2.5408711433410645,31859.80902147293,41333,0,31859.80902147293,0.2310354196238467,2472,0.0731419982532041,33012.86257123947,0.176319414615864,0.0638618957534992,0.4440256723898684,5348,0.1243953072949355 +1146.3031606674194,2.6046295166015625,33305.35944676399,43191,0,33305.35944676399,0.2310111741080069,2472,0.0728170129790993,34511.06227684021,0.1763380618651776,0.0641529365477172,0.4456140825924251,5348,0.1243277168927726 +1194.8423812389374,2.794597625732422,34752.82623553276,45127,0,34752.82623553276,0.2346047023195044,2472,0.0750715983182012,36009.81967043877,0.1780934036375427,0.0647026802701289,0.4487201674578059,5348,0.1250325882296142 +1242.190561056137,2.874882936477661,36198.51755785942,47058,0,36198.51755785942,0.2380288143589909,2472,0.0745638088274125,37505.41343545914,0.1830817994424512,0.0668639276494144,0.4555611084425859,5348,0.1279969101530439 +1289.2322795391085,2.995460033416748,37644.09406256676,48961,0,37644.09406256676,0.2460183419091554,2472,0.0781589584221965,39000.54671049118,0.1920238463708794,0.0703995343347292,0.472720902871961,5348,0.1316178245546275 +1334.7426974773407,3.2001163959503174,39089.643002033234,50783,0,39089.643002033234,0.2585415288889606,2472,0.0828103101578209,40494.20410013199,0.2052840014233496,0.0759346990692084,0.4910192353576452,5348,0.1385313571187177 +1380.3914070129397,3.269481897354126,40535.483557224274,52559,0,40535.483557224274,0.2638593114878379,2472,0.083480592285662,41988.20076799393,0.2161155397779911,0.0791954338933173,0.4994076756988396,5348,0.14022111717279 +1426.375376701355,3.331726312637329,41983.54762744904,54474,0,41983.54762744904,0.2708677525566165,2472,0.0870960534600776,43484.77414178848,0.2324301326915626,0.0842940374364695,0.5142982916729455,5348,0.1447400183459663 +1473.2940378189087,3.570307731628418,43428.78943896294,56411,0,43428.78943896294,0.2903184782082715,2472,0.09408323685333,44979.52585673332,0.2557255609176815,0.0928258444225265,0.5407247361299478,5348,0.1506107275624004 +1519.8126411437988,3.655649900436402,44874.081411361694,58326,0,44874.081411361694,0.2869752937689528,2472,0.0940019905348038,46473.932918787,0.2634646299711961,0.0942487105275922,0.5477415839423849,5348,0.1523970453338483 +1565.8220901489258,3.750631093978882,46319.49430727959,60188,0,46319.49430727959,0.2884309068326989,2472,0.0931895273495419,47967.867362737656,0.2630659183553152,0.0951972879309694,0.535008598145218,5348,0.1505238256167624 +1611.953557252884,3.8260538578033447,47764.89419078827,61973,0,47764.89419078827,0.3050043047132073,2472,0.0973534011740093,49461.83552980423,0.2825832322670844,0.0995575101999019,0.5602312139749096,5348,0.1548689228986626 +1657.9822750091553,4.041477918624878,49210.93877458573,63809,0,49210.93877458573,0.3006939207771469,2472,0.0977596327666402,50956.652635097504,0.2819004034227769,0.1000910849893015,0.5507110049979907,5348,0.155535171148554 +1704.2230117321014,4.118767023086548,50657.33596086502,65748,0,50657.33596086502,0.3072860547454674,2472,0.0972924664351146,52451.73114442825,0.2963697213940404,0.1038853946028101,0.568032922144992,5348,0.1609424033215854 +1751.6461789608002,4.180283546447754,52103.00220417976,67681,0,52103.00220417976,0.3143680892718885,2472,0.1023500497633701,53947.36892056465,0.3175066219448515,0.1108110875763308,0.5771099793424754,5348,0.1646309081253319 +1797.4855284690857,4.375433206558228,53548.67119860649,69583,0,53548.67119860649,0.3124170813439577,2472,0.0995470517742164,55441.55384373665,0.3075975499856823,0.1068982057874,0.5708020566480811,5348,0.1611451745280741 +1844.1046738624573,4.456562280654907,54994.37272357941,71419,0,54994.37272357941,0.313425916591393,2472,0.0993845591371641,56936.354231119156,0.311029117607425,0.10914029783174609,0.579972463174352,5348,0.16227489982136822 diff --git a/logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/trial_1/events.out.tfevents.1789582202.2348acb64598.38.0.v2 b/logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/trial_1/events.out.tfevents.1789582202.2348acb64598.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..dc83aed86bb4d032262e3af66d21b9c9ba2dbdf2 GIT binary patch literal 94085 zcmbT9d0Y+MAIC#UNs3U`gp#!=B9yYro;__8kxFIDmLgFoijX8EB%v$`Aw&_`DpA=b z6iFz4Pxt!GoX?z@+l_yp=lgk`_c>?2@0qzX(;yk~znm=7w+b>c-p|h*%^lpbk-fW} zlS_}GOB`H0`}dmIQBlj&!NtSX-O<@~X^+{i?hZW`dpbGyaH#(89yUweoje`fW0f2$ zyZucoi^~|Q(9L^9adldk_KNKkWu09;Jj@gu)M+B4CA%?A&TLdeMFmB9kJ)z4cJ3b5 ze?}YX|E=BY>-%28B=B#owqi#`g}Lr_b8K8(-535p;Lo2wgCJl}ru)O*e*=^bYfe!J z`SmxTv!b?Q!@16`_IA!T9-a_8G&f>hY`8>C#nE zS5eusdQN)I_MB})vRHMii!5d1Jr2T99^YVcL^R@@r4H`@A4*Awxu!6bb;}m~8TfZ7 zwj)c673$q%BJ>n{C^lVU=j=4c&eO@29#89AzXn6t%7L03Q^YBm@SjJ2x^S*xr z-@dy&U14uI6WB$uvtlDp2MAw zan7!D=Q_E}g~aMw*iZ|i!@@i*yu`qZBaBg+o`e`8WDvCpb{ zI6Ygx>VAMW;$Fp6OW72U+%m0#`4}e`{r>?-Js)Q;HfjeHQM%dUkh*kZKkLD1>^Dv? zH$w`2rdmphsXhmf0mZ?7OoT9n3?AYKwq{)f3N80vVv2e+g;gkrV#>qW(Cp$fFj?&?QfByWNC_XGdMfSA+ggXpqLZPRP$3% zn$>A|q7o=J?_wf^DZG)QeCJPZps3?NSWF>DQxs{);&G8zWaI`Exj{@dKLxcGnJU50 zfZ|vf6Cq3yi4^sU*X9Go>rM`0iuyFglxLi^_>({OI8dDG!Bq28{N|qWoayO34=A?w zWg>(rQjkJ+!@_Mqp(O7wrjVy8<{$Zu>b;?3HVY_jDl^qmQamqjSq2m@TQCvA6a`4p z&@pj7P}KQ;R7}x;rg;AT4^oWytcu6Q%il~jKLxd;bPhZH28tc>$NoP6VG0=>M;qNP zRY0+B^j&d?0!?wL;wM&X_IQrISUriU=BJ=`l=0fbaX^t~#Y6~GsNgsZKdlc1iY3oW z#T1G(#oT^Bu-etG#T(dB!{0E~{1nt$JW45B2o#2&m3R9#Q8diD*C~R!4#1xHaikM%Yv3h&2yh@;G zH;<|2r=WJ!+20bepR{u|Hysm_nJR(7yKxt7jxOd<+yPqnTmCDAblmi76V>6tnld!fI3Vn^%G2qc>B{PeJXdPrZ|} zfuiL`CPJ9P4abpLxNs0q99wo>Orb(koO$vVt210<@d7{2o2izP;>({9 zm_m)F_}=^xRyVu8@&-^e>CIG2Nipid!w*35qCXQMOc9CWkojGB11R)YEf!NWqbY7E zJ;Z7+N1NF|adkaY%}+t~sCx+~zXFBj7A8WNA_m9t?OR+0P)y1Q6H_#&DNfaUjMc^$ z?)LzS#t)flehO+w?M;?l2^16ZmzyyAP^T#_Ij3WFr?cr6KykpBsphAk zc2sbZN@t*mHfJJ)DN=A8*3;L71BLOFx^+aIR)eOPw)z%UugM(NA1I2fnQDHDD()5Q zs>MNdfWmPu6Cq5IhT~|isr(8kWIa2HDO%DL--51U_3>>UF+eeJB~#5$L9IpS3D+9| z#fo)IgfPWD97krG>8e2Sde{gtMJt+O++)s4Ug`HL3n&uDGSyO2Z0YW82^1fuFcHEO z*+}tqs_zV-SaW58n4&dJvA-|ps5&YWG#4oL-D0Yxq?nr18t>%nzRyGmQxqUY{G)@B zK#{w3iCA$_t(I<|n=BJ=~)RaxDIsrw~GfadqMFmnsN1tg5Yw_dzcQJ(~O=0N4Ie(eZ z&3ZpjWc^{P`6;L#H7;Ss4WRfWU-AC|2vf-5o!0Ek)P+DXV05dxB44zpDLw>6;2bPp zHDfeTluTr*`6;L~kY?=;V2|oQlZg z<}=k&Qf%&hGZxmu*n^1>rf7g`q2Su_98mOmXD6oUKvVQx7K#)`CI-wI zOreGpbNY_13u`g_c)FNEo2D2L?~4@4MMWckVrC*!EhR-<)AV6Lk$#bh5T;N^iZZ|P zErCMz?I$sX4o#uL86hA*R)%+CHj{4rg67M$aR4@_36s?is#?a_DKyj#P zlX{{~+nJ^)9_5M@HxHC|2Z|o8nQDFt%H63EdP89?(mFB`!W5cFp(rIwQr^V=hO4B1d7Rn4&995j1o>QnbC6 z7y%Sjs!TOM1$A6>J{yGYF415jgekfq#p>^&n}A|s<9%X^ZZyR$=ix|kB*MB2P}EUp zs`)7>>%E=*eLbwja7`vcm_iRJI#mvh1&Z9(SH%?FX^M7r1|o$+r{0r+VrD0%nxBH+ zQ8^(AK(V(w6Cq4tfE4?RtnvCH#`3k8q6bYeWJqtMaOsj&3=}z5Of^3Rb##wP%lQNp zGaQ%*VG1LpNborD8P>vJeFIrhr|n5o_?_v26yv_Ai-IzynxBF?F687hmIB52DkeghVhB=188p@cio%DB z#1y?~irotBk)pp}9YdgK@{FnGr=Zp%CBVE8D70TQ5yBKBkRs#o?887&)-qH~(VM1d zeoq}Kf=3()0E(F%m})60Iw?&b3ltN(G7-WQ7D%BwwN(mGv}}1!OkqG%ywYrr6b2pA z@D68=7E{enLG>v08IgESJFzPhAxtqIDUJ@EH4`Yh`8*X<7}69U?=(S*V*$6Dz*=~1 zVygKms2#QJnAIX!3#)BRgfN9AQY<&_oDLLsOn!?g`p^{p%2kj;y=lpKpy)V)sg{!B zsoa}Dpm3PLLvUpQ4g`&)nE_ zr7cio<}wk&6jn$fGx+m0pjh}~h?v5NrkFdi0a6TqFg6V+CKWN&{1nuV3XN3V02Bsa znFwJDTcq%NSMUrdE+ozsQ}m-L?s~~1#qaa)?*m2NMW$Lx3PZVR=|B;GgNYEPn1d9T zGwpPNqUG~VVv7DWh2?%3q`3U%+7O_4{)VaMr=WIJy+vEj14YIsCPJ8E9#S}@^}&1Q zqwB1sphBn!96ZUbl!9VDC#z0B7`ZNks{pn(I{Ap9)b78 z6a#1qxzj)Jsl)YpnRY~wovi;>|>gfN90Qn(rmLWx1k2$AcE%yBGFQzb|DZYf3AjRQL zFYp=8JN1qKU(HWJ?Wm>`ihBS>$Hq*AFvUuw*xS(HJWw30nk62GDNS*@st742uC`bT zYmq8v{r_ry3Tj7X=62o<6tk6?2w@6uq&RB%IvOY(ZhDKy!ShPm%%XQlA$#6(9#9x& zGS&PP)Q%cD#AgCfWItgdgeiQGqL=TLkw6jM`JkA>j9!a1?jMliTUYJ%KoQi7sphAk zcGRldPr3kwlQ9z^OtB6r`s7bD1PXP9n_>!cnqqO-d!%^2vT_Abc&IYf{1ns~$m-o~ zUIWE>4JJaEg7eDKz_8>Xpjc#FB&HZbQ>;^ZjTHAM`s0q8JCv#Br>NleD7`*PtAV0y z3=<(t;g4(Kop7)*P#AV;Broc;Lum@1_*Y2LLodS-)}n21rkbDPJC{Po{^|vwur+2P zgef*5#oN1q6M-UPr=FN%7)_zls}L!^83*Eh+Q5BGH9rM)Tzpn9!(Z)XVwebFia?}z ze{ZQ5P~3K#B&HZnQ&c{Ai4=2xf5bbS`pcPWehRA7mJA(@&kY8yWg>(rwjjkGZ7+OQ zO>zGcF~ta);@3RRTExD184f#Y))A(fpMtuR^X=Wjaj>IypJXD0DT0u~Xp_1#ti?UW z-C~N7G(|}V4n@^flSH6s-ju24r=ad094-k_2MXC%OoTASHl(mlbtnOfMz4~^6r*U0 z%P*hdj!JBQcMnjQePXKlDX25^;g0tX0EKHg6Cq3yf)vVTjtW5Ga;QK|F`B0Ebl_0j zYF}RkDDKBG)%+CH(S51TR=n1Gc!r4(rr3cLgKqQ~02Is2WE+S&tp!an_sUaTi;VcS zXMkeDD5jd9f;zf=U8m;3TF6aeB7`ZzkV4k(Lp`AAw@XJ%F@~nlS@aw!`n7j|2NWR( zm}-6s>bSW1JQaV}`*4hj5T@9T6d~(x#skHggQLV0V`&Qi)_F*Axq;k1pzw=js`)9X z-;#c$OmTs=h)7@}gek(2V&{%enLy!FxKKzP0{d9Hd0i@XDtK@C1a+VpMvU92NoFPT6{5Q zB7`X-kz#veQQetx0?xTk-ChD5T=Mi zil_nVV}T;a?1`9SB2CeH_7kL#J)dF?6az*x)%+CY+_S+&qZLm9MdTzVLYU$pQe<8U z!@Fw5y?%-*Ceaj`m5-5Pzuzrgpm^S&sphAk&S_h8?&}W}>&=-6VTx#^$jaS>zp-9P zZKELag(Xd){`DbJ3>cu}1r+V?FxC7N)K$`=$d65dV(23#LYN{3DZDPcGKU>C(tVTxF!c&0Ph2Pn>lZxB;Vr722Ran?fn z$6S27O70L-EhRIij0EMf4GeuFSok>$zt-Oa6eX^6#dPC+j)%+CH8Hm?B75u&X zjyn?}OmPM&j#qUb0~BAh`iUv5XbQ6y90~*ZA46d+T)Q&W{1nu2v2C(!KCH!q-b{or z#W|!%>o9aMP@M6#5>w2gDNcO6gKM$l!uS}V*dD-COG#ns_XwW@Ik%mO5T-~%isVif zc%Rl@-&;&!O;bcEWgtcVgd}I6Xy2cy=BJ=`ly*S*6tew`#1wWk#eMzTxE8m2 zcW(z2!_1g!DJi^;Y;6D(Nu!tuVT!9r@oMtp(Lf=8woyY-r?sakj$glp6zSiFwRfRVY9!XI#8^C&s6hM z(C*f^(SWtsT*^cUQ`|rbn*>w5!a`bDZeQvnj*uBQHSn{7vI?6a4D!<)J~cPJtil#R;x)G&6@k3t`TEkt@8$@y zwLD`bfXvlp%cTUFcHiv^fEe{+GlW6Bh5(3bcefn?5|(fLMFf(^^SPWim7Njf?VpV% z0CKp9Ete9c#k{@^0A$8DHbW4^`vQVkYPO!y@9(+>$oDN(XqxzI(Yfl6QQT}S=_8;Y zdtO1Lt6zMR04XSi4H6+yUja3(vj{&K7w(nnH0bYOM4xBflE!A#^cm0|moMRnp4r+A z0kmbAY?_4cfZBOG;IKlA)uZ6MjXF7O&_8_$R4v4%1$_0=q>xP%{t{?>s|$#@Ty;S% zG^l;=*&N`%036Jyi{$_=KyJw>?;0K#U ze+)FhgR>cj9r9=bE2dGWlCv3kzXlo@aSM0#dQ1c z2(m4`QC|S@yUCXGgHY#n{rntf0*L=3HbW4^v^9RP_gURu+5nQ}TGZ=&b;m!)^M#Vv zXX_)#=a5Gu0c7fOwp>aO&+-=f0P@g>%@73fHb#(_dv{(25Rdiy){8)5c|IvIa)S(l zgwK%;0gwYhY&ky&^}N%7c`t4ONXKwCLl7io27+8TylWG*y+;Y!??fQa+DdY8Xtylv z<a#)8I+z;X&_7**z`-k(+0>dHd?>0dUXkeT&9MguTeM*Fq;jyP`vF`{^UGT{ zK~sor%SH+|g?o!}`mFRDuLTG1=)|Vf&cR_5MjwSnxhn4A4^e-KRom7r$Um9Ivn$f^#C{g>zE6;nP+Cd3vqM_@rueh46e7LbpvRf=r(l za17d`S|(f04?>+x9QU3W4j@Nz*bG6C$Up>%e}2;iK$293o)v-k^Ss61Xc7lxrK*=B zfVj70%cTT4f9Usb0NLGv%@71J4M&it3O8f{#P9h2CKsw#q>|_5@s57QxFV;=El&ZE z!ROg>eh});MAV6$x&YEBmCXk~NK=;GQ3s^y zo7f-`66N5}*UYek^qlRp9~|tqgUzVP!B15!aYPEQ+=c_1%Rx3xf`dQQ_@TS*n7VY3b!;QEZoBn9#3M zuqjMCi_HWcwYbBWcv(3Zo6+skWRPR zaw$QgG+kE!$j66lh9HP$I)d2Bp8D^Lwc>xGKU}k{;f+Db)#}RTd0$smHU~kR2JS0` z6-jKymh*#9>U!{eqauKGYR6^>f~4dkNW+1vMgfR@3 zT0dZeL`al_%fdQg#-f-z=-`rEHlrp7Z&n$EliZ}g`F21ve#xduaPTng={PJS_g%fg z!5u%cLI337p0;UC!NK3k*hFCm>w3Myk%h(GRs;vX{l(_+IykSn8b)Yjd^`yb_LB`{ zHb5;MT)%VpDRA%tB{qw8@JHJ-xEY_-{}=%)wnB}qm*C(R=PzN>1L zriS;^Y*Z(3uu4ZZwPpvmllMjke|P-T7#zI68=EJUgR`?af1yr&`O#$PZqp6eNWrF% z_64VJ@s#%=;9x6bHl=nBZk%DZ0UBjQ(Fnzh)g3>D=Z)LBH*exGJZgkm8gv!CZ)`b# zqoi)(W)8}8fjKe%&1MMBiOf%&lXxeZAPX7NT!NauS?ZiE1c&gr1yy#2x2s5SsAQ|LuPZ%her!9nWh-ugY&D$fkq$)c_2dvb+xZI zirGxGx~^I`z|EN7*W)&9#-_O*;1lkj*w7JEl~ttAOYI6VJoR2r~8Oj#&UQ-H9#d2cgszb?aRt02%Gh zW(b0K8zV@sakIJsNPe?{VIq)+JTDAnoVV}k+2peLIl z2r}LrL7J~sIt(C>mcDx_0!eeAAMjQ^z;STW*uDj@A_INca()o%C9`WMlBNL2l>jzF z5JYn{f*c5(_6geFAgg?XE7e=;(OgLmws??;I*9S(oF9#x#|DXzsLRLD=%46=WuFe% zfrIzBu^BZv*d($w?v{6EhR*=aa5}|@P1!5=${;%;AXo89GoA( zCJH+^TmLwYti#po-r(RD+u0mm2RF5SiF2MaDb*bu>>AES)Y8G1)_zX`2bUaRvuFob zert=H@#Om_A7RCs#j^Dh9IRsB34`Vjkf{d_K75J|lGef0@HS1Dk_QeBO=44Pc5vPP zsp#N6YU)S8!Aq{Oc~UtzTX7uj`S*?XPlu*3FP)7PYzmg+aQgBm_-BHHJ+s)9+BsO$ zCk$^G?mX_|DC#fCPIRMe*DxM=R@*jO290v%dA6LtQBn?e9=C4a z+n?(IqMlLVE&>T#OoQm%;(&Det}zW(cA$-&E9>JI`3+qtrd!VYe)8jT|x z{6>E`I(R9Y!|UMsho0b^TTk|$1rEOF%|_JH!MEM*JA#8}`LkKHg98E$aWk%d?okg` zY|&P>UV?+)qz=HKQ01uo;NWFD*dS>gObyRw^k=*dUb&Y|t=YkI7w4gaJNDiB1RU%Y z&E`qv;5lu4aL;#&tuqLk!nrs$Qm`q^S%}lucli7r;9&VPY)b7MT$naM4jN^5tLHyN z{UvZY-6)f~9!H)T-EMq>uJUazTh8AosV|F7b@}K3bF$Eb%@CXuJ$IawuLF-614wIy zTc+2l)#bg41{of35F(jF0|)7t zCspS~AZ6ZlXp(8UiAd7VG7w+2-ui;5mlCDnma4&hWyJs9zPmkLVd6(3M3Cj3H?r7o z?fx2APWhz&7O}ir$HQ{e=LE7m+G*1jUbt4+Ow{wUP-hfAQyg{!%b*=Zh#`7v5U-Fo+N9&e%TDqCH1O}l+m0+PR%q6b z$f%`}t9E*r>;Dbg>+Abo;f@IrMk_hElLhX|sP)QiVBK1cAOa*fxvRD%COxz@eKY*; zP?E0w+N0n*mPnFT%G3bgHz?2_{5MwS$kk&CCR2#mnyu_&P=;3Co@9P<$lpNo={!ZxlC8kfCoRkD zdbI{pgLqi%mT}IQ!h6Pi1D2UtL_I$Xb#0k7x>*BYNzWxh1X=upkY%q;*Hh3u%AQB> z5V6FCNK{&R*-mV%NAy?z0IKiri6jA~4S#^0RaK#987S?2DG^eW(lQh|N(*b7wjb!^ zeiCsKls2tW3+WD6PCEfgbCx~K&eK0BEwt4qMNnF{5)mn^G-aDzIJT-2nJYkPE7XV} zUZs^hO2dht*e$^dls2smkx@&f=~*B51f>;rB*JK=X(V&*-7J0jNg39yes>~3g3?si zEWrJ3d8|$}C{4kDNRn1*)BuMh+4_Rgl8uSjnw2(apB#RU;MMB>+d*lbLx@1BlqNGW z43!q7vT{1K2%|AXreKS(3c)GtAJuj`C{5dvh^d{@R)r5e53Td$)vaNoc608(cGF(5 zouaI>tA~e~VuLzOWVB>g2OmYGNfSQahbH5fPSo?aPU>n?|3><4n3t6gi4ehgapTO3 z+{?t@z*4khQ{x-e8i)>wpRFp(LC+h^GBz7+;I0IBx1BfIM6?Mfqr`c_^M9axZ(?QP{h7ch& z>3PaTC7kKTGmf+ax~CRIoCG~LAJ-P?7Txah3iLc@Dv|V0dVcFMry=M$+J=Y}*7LdP zJ8*0|P4k+Ap4-hMf_U}par_!i{C%_3^KH5E z%b@3?okXBidQQ2qA9w$b)3%+V-R<8;WD2&3AyGJmD^gee20bT66EU^ZbDv|m6QFfw zRXy)2YB&A|d0J=6SdIpw+BQ;zCUd_*K3gwk>oi~PI~nFBt_cw$I4>#(abAq%Q%(cR z`py$}iC8KR^RSqV;hbu0m78e~EVm7adVUt_Ds9}_&i#SKb`TLF$Wn0_S*|^`CJEY?6IzY1X#ka6ZQNo)M?YETgEeiMJUsusFLl8B1ZbNq-)Xo>fu=a_(=C+HI)HR;(TQ4VK1cF?$) z(B?PwBjP0Jd9tB8(oJcmxfk^8HiSs}Cq1_}zT*yhRm}U%cf7PuL5s)=ATkA8L}WZpq07<5-9gXs+lZLj>3P%mM%$rv7FuUq7PXt~(>$%S zYkV{!JqX-&8JbMce4?JebyANRjd|4U56sKq#YBkUyu_czc{x1mx*f2*h~3fQR<#B) z&hoGn@8Nu=&ESKjA+TINL)7!LP#Q2bc3%T5VV8*zL6)$y$Wj_Ss}NXR&$e18VoABc z!{Rl8!*V!w%PU|xewC=_XQ8e**Qp(e1(p|gh!8;*?+eJ{tGf-q`C1s-^`(d<;W7_P zr0EgdV=Y!|Y66QvBvCIVi`>fJZGh!s3=ty8GVe07+|h~9hekN@T169|nM3RV#((~>Kr_d6f6|Rt$w1fb0TCxb&!-x4zSD5gENLC+IsX}v^iO)8^2?_R^!)ra5h<)^1LXjm`O}S; z;H%UfONk&}Jsak8CSD`L8#M>?yj_Ecty#|@=ABW`uYA4oK+kj9 z6M<6cxx60d>kS#LVvV3hG}R?C1zSWkr$so%+japxkJKY#YNu!W22b#@ov>dECf%-9 zTFf<`)_Lm)N6(Y`AJT#*^Q*oyThHG*DLqdf*1k8)OWh_!h~T`8zlNK%;n9FMptKzw zx*r#@gx}<0sTjw(xzWS@B>q+2OWlcjeilk;*C#&g3oIplh!8;*qnpT*T{%PxSem6} zsHIo0OHeuw%hY3>tLNzG5M();Mbt~lQgXQY17JydMuZ5mXr?1e>xLtSLG#GVaa|~4 zd3IN#(n9Aiz{c9qU~oU!ZA!0+Bz~o>-*On0wn{zQ0F+i#OoY^=w8(%e9FwhH`9+}H zRYt@~P};e@jWKD$&arDjX^~Y#(myF}L&f4jptSySZtTZ5 znRr4Mc0C7wGqM>%WD2&3q`SBndGlLagVOXy6EU?@+Jeb$JE3*v|CV1WYB%dLd0Jui$+BNG?}Rl8nE^Jt&{q_c96n-9cZ0rREZG5d1;V|^YTo0%P?TMYS`=7UU|GDK2oYpS$wijC;SP?_2tytHUW!;!pGnkn zBSmWznCZQoVxZFYAd*B>)D>rL<4Ckbe);dWpl6$vL`Y3~E*(&gWAfd4$q4Ay_!4mv z^co#nv1?YK7I1$9FXUFNB z^CFLWZ)Sp?siIf6!qLb*UAp`ye)$WluFMQk*~4OXb(B- z0WG5LLn2eKMfg9%#aLFLSfk-WX zI{YylMAj`5h$da_A(KBHAxnfYCyVZLvzKDkSX(jr`nE&UVvssNEd=#M3&}TSXvJC&T@BpvgSmO4Li) zI-Sb*>%zPk?jk}2=VinvoENQG^$md~XNFmW%xVpUec@qQdzkYuy7`>b_}gheE0!RH7>dlA%ZM=Uy#M~gS$Pj7SHV zU;R@%P})XwB2rjsib;!cYw(hZMiD{0O8d1c94G$g;7S`SIr0%_mPlX`%}TS-nTkq#d-sDaC~dPB5h#_?#y{iyrM6zmZ&pH!IORiR3bu&AAGjD3x3`T4 zrDbg-Vrr+f{=tneL+d=*TkeUd-FW}xX`TBYac(?rd{ej^n#|_`L_L4&q;3eW9kyo` z%uCauM2O(L$o#~4iC8;&FR=Wu7_51(T4`lfJS@NK1)-j`Ggc}9OP{GkJwFR|_3R-t z7aue8v?W3WS>9D4%gE0U^?}9sh}9ku%e%F8=@+XsEIC)t3#~p*1eU@CqMo0Hx_aI+ zI4}{`CI2!JBFrK)9R7;Hk@QyhIMw;1FA78~dFy#t<_B|5n@qGn1OZD*0#VP;LhZ57 zpIpWQOUOkcM35z~F0!;!?0yXz;RL-kL+)4au{8g>k{&^Ft2+=SF@DYcWTYBEB#Ee~ zD^Ax6f3$?xzJhO{=hs7tkec*dzwj-N>E6pISD@=Vj);?>=MgLGV$xHYPpd)C^=1)C z|DKS73hP!cgb0wJ=dnE=VBeXy(D(!Bxy2qLNm@Nq16;lI z>p;-6<3S>}W0;STkL!~1A#p%FVD!ZUXyiO!C1zW@jS!@yV zHf!ub&tonUF}2h4OO3Pm9;asKuJc9h=H5o0*4cmjHdLDLPR(Y}WHuNO_57`qQd->I zE^;t0-%W@R!FjpInU^&8E*C**Hn#2KMJ%bCd00Aq4n`K6?N@dHi~D?{UP_i#&*t9) zmd74Mh#*U9eO%*i&gQr-$y!5nv#K?4E|`a<#GJ#jO|?@ru#|Qq>iJnHJwI*Q8{e;e zY)FI%vYcyxEWxhV@UF(hFSlGpEU}?HETzwO;<_|5NlgG2uir#HKMQpyKxuC~ysI%= z@isdML6%rWWLb3KQ#>@nZjSy*B9z`&vEIxyFt&( zw-b@Vde%#H!LhCBXBG{5mfu4J@#=ZiWX_%V7ANFOK+kdqiHut6Is0JYD$w)WV?-FO zXS4cGP^!C+#`c4Co0>obNYL|+=bRta~PyP_49(y*w-%W^zt7B9^_t-=4j@6ZQNol+rq@ns)-0yGBHa zAWMicvUs}2)dQB~5Veyc7XSS`EaR27fVfBesIo`T=yhOxF(DU5Be{f(;?S9k&Jzq>ClKx50Q}3_D-)lSH zCnAOQ-1eC>&iu>x9bJ#1>$ zbFcMYsOL3qaWg>A4Hbw$sr3B2z8dOz>qX1G(C%ia5}AT6VoXzP5zdz{n1P;6)rpwe z=~*H3n>w`4JJYOJirP&{3{UG+d&RjS++F*1J~Wvqd!nAdby8P0?tRvLhI#ohp9m40 z7dJJWml;pxuK~-VxbgQzEU#jDSo)`Mey>%I4Z|mdPn{>~rDPeNZ+sV6DpQFNL6$kq zk;UNX#bRLjJ}toLQMJ;t<9S%ZCvjNXCSS+5Hy_#)_53W<4dE_|U+saVtuqlK$TCeG zS#q7X1_8_H?JI&sEEx$rEDrTTP|xE&E^>iA)*_Oq=VzhzSZihFX~3d=j0h2Av1o}b z_G7GWLL)4hZxk(J*_MhekoARTS_E}s3<*8-@wuHi5W*Awp`> zv-eDnp4Yt|VGMLDHV|s(SAiLKMQa8L_qGJ_Tq^-{LZMU6W4f_X9WBSHk{#k4iHPOY3(_%c$| zit}?sED6axEO}i*k!Ad+ZykZfYa>z5&qBQ`tUJ?59$1n?i4Z{+gSNr%wgQ#5EiGOR)@^@z5B-*F;ITBT6~d^^ZwD=6)E0ufuY(tPAMqS9<{KS~0n_^@ zu;`m}uF`T$Q_O(nnGI3T&qCcqbsM3W1}yXE5g~#sDjkt!PlGwvfh8(YvqHqOE`x_< z-W<*i;n!Q=$O21d3Q^C`LMhE=!iYA&l8{b>2(rj(Bg^!2^U8tc<;yMOv#XU>k;%jI z^2Js>)tDDsiI40>e8sOq<(@H?k3jRcF&3ZmPax3cjaP-Bqpy%D&i9o6JYtUx(_nv0?XR49vek051;a|6y}#wF>rxa?gb1=c?2ar7r_bPDd760tbFPRbEuV+w?0wFU4s$*y zmBYF`$tCLfStvahOo`bBERSCjA%ZMvJ(0z?u8JZw!ncRJDCAb}v6}@F_1xCF4_>K0 zlJ&)V0$WZINg^sr&rf_fk5koAI5Y_KY;=JLsY%b#mpG48$@bTa1G?~QM4SXY2Q=V3 zPIX!7${f)1i91BnKk2z;#LZ)%=aG+zNMSwq9pHvDuk|3dGw8YGIT6IG=MI;5;nnk% z$R5-hbi*7D_7KU>;dbRR89m)&~w%i&NDC3>BoaX z&y`g~lC*lJ2I$!;U;ya(kz7~yu&G(k-)4lOo;MeDi3C0CDHDNG={Y{P7wXxo_@gtl zh~CYKOu-g0Tn`t+%1_q_ZV1P=A!2H$XPYAqoj?N{dZkViwVULZJgsx~49*qj>htNg z&^pxy67~G8le&67x5noH%uDHTB1CXrlJ#+3+RL=-1uSPzAC3{R#J}ZXi7MdS5Y9N) zr3zTST_o!HStt$Yn{D+67S)?Xh#*URZ)CCWVlf$5;)BXGo>pt%;0GR-R$aNL8Y6nW z0ha7NM7@+OGjD9#1T43si4Z}SgNDdb7&sF@v7-Eaj-!Yr>@yF`hO04n+O%h%Ki&r3 zB;&x=^RrNU?CIj^!N8KEK!gagg!M%h*QE`spb;({&~%N6CF?&HaUX{ng5yO_spy%k_L~PA^))=-2^}Kqlepk?QaTF0Km7bgZ?u&Y^8d&8GEkgAe zktx_BvN$c`wcN50(DS%cL`?1U?AUf4{so@Ji&Ni;+D+hBp4O>5lXDYwoKctM&}2Nk ziF*FlNh!^`eO3|7i%9?xA~-LB{cw{O|60BXlvciKcdxu^rFnnnVL98FbE7`x#vEl} zdAEV6=Vzg=p07tf#jEGoU?N12#oHKJbo1(d2NtuXMlm9mvY$LGHy3bN#zzdor=1;r zhjYkp?O4W7nX@wuKbp$v^V>QU}J4F{CqoDp-cKi z62H=-x^q5%k$Tu`HYn{@KO&?ir7dsp1;=#p-j$C)mo}J)lc2N*H|k^3(Y3R#gVF*= z6G{K1w2e_(-9c$ylZZ%Rr5P8mz_CR&45$N28$FW<;#JyRRnC_#2Yz3B7L?{Xo5-l8 z(gGTN$OWZ^FCfBbrKu}%9!k|3pMxh_Z9Iqo2}%p(J}!7gPVWmS?U@&mB(2h@0XjUN zPy|Yg^C4nuR+_8MepH(G*=gHAX#oL5pj1kWNHjsEz4Q4U0j=9_JCP~aA_kjai#X-( zKE#CYI`pceGD=p8b9?h~jfphizIbhKtXqYzEL_I$X^$pOPWjQa2skT`02Uy-WChGZFs4GtA zy=psPT`F4=A%ZMJh9ZmElKNjjV1B`7^F=KAf%PO^aaI^kL`m#5o>&M}q2WZ5h>E)6 zJnXv%FMQf8IO7C*);dIl)THME$6q+6f~WoPd&vEd6LAvs+~sK_Oq%%p@>kIFwKGK0 zKk0eI#0R56&r2^8k-~c3VzUm%wj;Owa?o?-4I+qF&z@mNaN<>-Y*`O_u9rz<)KbrW z8MjA*o~tkKPf-15A4eD=R|-6Jsb4kJo?yCJKzlHx$+f}B(0vQ0S>Tk zpaFW$C?;ZS)^qBUXwi?A7nEh2kF z-BzIIU$W+Gi>RHRUpZF}ht}Es(}e(0yE(Uor**D>&H1Y}RS`1ykwWY5L_L4&q%;tz zqILx4MN1}`9faV#7!AjH>G`NzFt7}Xny#2%t%2BWJS^kRaem|6pOA@iJoy zE6&CrJ=K8aV-gV}$kJ^jvM7HUg6pFBIen~%C2|K3%h!~>coTJ)w|ylv|D+#8JwFTe zJGx%gY8zm=Qa6JggdmINXkzjX{=6D|ALfBb0q}C11pn`5%k85%S&;o-<|^N@Bb7-~+Hi zg`bHe5f!E9zWSWc@;ZIC*$H~?SwV!(;Ut z5gvqi(UI%J z<$fL(zbPCIB>U~`1uR$9iF$q(>Qtkx>ktcI>D8VH5oDHsFsv(gitNqNW|8xw0ah&P-#;X%};~U1`i_wrBYfQ z)lRp+zW+B{BtD#BEDljO1C}Hi6O_OeSJ#r?i$8*YP*biQb2;MD6C)f9;04 zA$)N%M+1)i3ZtNPzTHUFOW8WJgBo3jc^R~w2oao@SDbl~^&8LyG~l%9cbteN`#2BF z(cPS@=Z7KYCxE4V2T{+@LS5O^scepqAbCU)A%ZN~Q*e!!z3g!ZSPV-_TfV5)K*mWP z7I*98*l4H8rrZOT@Sj9IKMQrL5jeB}Z`7O1eqje8$dWM)S+*n@m;j6G*C;m;OG+XS z%k%Y|o7=IuPI%hn_lKzGXQA!{80k;L-_bJ^53qv}WJ#HUEEk`@&4xy}w0l^(h~?=y ziF#i0iSuXLmX^;ff)#pcL?nr*s4GsdR+muE4uR^?py#oJiIAG~T;i;VyJqPhMSKzD z+Xx~~f}T(JYl#Xxb*N$-=(*=aBI%#>Y}G#Y8|Ybg1`#Q&=g5O0IJUXVMuvl)H`^0I zym~$pmx2?2t;LTh&~xwkL`E(3tfas72IzT_I}t|f`SwAMR6F($`USgktrrm>LC+gb zbDn@#U*kU;^xVRSNRn31)BvCB9E}D&AKysC)~shI-E*ktjFzz}pyzYjh(M|IT)buu z>Ur_=Q+NjdaW|1E*dk_HVT-up<-QN}+%k%Yshys0JZkI+t#icK`maRoCgD6!>-4aS z#REDg;UoT;uNPB^dj8f)U8S9?H2Mj8US~&y2+m6aXI>18x8YZiv?i?{^s-uMF&BAQ z!Zsd2mR3hQ<7e?d%_8dgStzB&Mkx;lmaa}jh#*UhHLmge{C#-ZG<(h7ND)i;6&{vp z3g?hzS|1&J>7~vlqMo0HQd)7?_dCGSe>)K($P#XgEU6wzqhOECf4W`fRrR_AUFTsr zaEr6Yb}G(T1|oa&hN$Ofq1GjO>9U@{5?xA!2(kp(Bg>ZqRjZ*9j>zjZSj6)9rbInI ze#QAC=S#HgOM&Y78zM?ul3`GI&^4KRkxb zsbhf5Xkbw=BI@~Bs4qG-+iCY0Se}^?A%ZN+9Fb+;sJ#8a(mkY7_jR?>itq8ToM@VW zEZMVTb%14L1X0h=LY+444l@4=ECJC(h#-seJY+F&dj1Poaz1zji&zRC^00ij;oJ%6 zcHjF3u$=r()bq1YcQvek^_>VTesw+BK?t(g&PSF}>X*(#BP?||en!L+{#c@(x5snz zY_-lEe}34rh)5DqQF_iOzlwV9T%}$GdY-qO2&qZWlgb<5uBjW95e;-l*Aj6O^!!1K zbM@@MUBegjd@+DX`X@aX{>)hhdfv93h!oay-{ad+&o@$Zl|awldx#)jJ?rFhZd>$x zxflPu{EY)dMlJPRU-|SF&~yATB8=Aan$HHfEAQ`hEr50Vbeaf|py%f{oHr9}Uq6`x zdd^NFlBCr$HNZI8rxwq`w#OS*=7)^Zx02zs8DP6SG&=c$TrsON%oqe;*r zW$H4%0T1Y-P4=#U z*4ZnbsF$*JS`Ns>PZNwhPlO20i={Kp%S;u+si1*du@CIuR4XklkB3F$H|JC%Y(-!U zu+%+I)bq1YSDZb%OsfxT>~)<85o8(Rf-Lg#uRjCJqZu<2MJ&k$JS0SXwt8JcUrAJ2N?Z^&*0%nu z@B<~zHbfG?(lTGAqted*JxBQDD9Fz5yY!BBfFb8@eWq9mT<+n zFqp`wrPAhYxH2D<7QKrIqm}kLq(AP;CsxKDux@+z69E#G_Slf~WJMdwEwJ9?+ID>G*d0xs z`CBJ-W%K*Q^l>mR5mrQq;JoN@=B0l#S^QhP9a5rvMJxy3@vvn0MIpDl2I=Z3KSq((tNXYqoFlc49lL)xJNr)WIFzrge7 z1CjJkdJZqu4FNqXe5%j#MUIDug zYN_Xba)Z=B&)!Nz7_H}${-(GqJ>*p)VBLnO5djkPd{~L|mc@>=ace-&2U`(I((0KS zV8-UzPe9KR9f;VP_3UbriF&TTvVH>SxkWc3P%1s&p0pPAtkb9fKN+#afXEbV5rdcG zV$3r;@Bywk=NJ<)wbOIu>oFgpbsm%Tye4Wl-eo+kvv+Jfw$3kqnz=)hNmn82rEHz8 z-1c08*7>;=5h6G*GF~_@r!V?h088gC!&Tl_YoP2qAIpl9$TFqiygaa!^(N}2WI3I9 zy9=;54<=rnHmTASD)22RZN6!Jv87HEipM|>Ow5k7O z3a~hN5+Q;t?^Yp;_4yI__iIO;^T`yk zkv-=OB1DiSZw<1@HH|ibM!4nnffhy8d+gy_SxGPO+_<3Pq5RXhT;#}xGhy6vxtI0<@gezzminO=D^1FoLEKNCs+ zr024ceUF2lFMTH>h4sA4axaeUsCADHpy$XxL=dl@hYjXjJrDcwr48u$i~K=$AJkIM zPWv0+@3og36JfNTrxbGD`o0`|x+AQcoH`L8LC+UnTHyX(vv0^b(DU}TM3S_6rUp1e z=d~N?xv>rrTeF@!$Uj0oXR1Gl20icZNd!uz=aLLx)N`XJ7Ut0Ib{Y|xf-PdcH%?)Y z;ty5Ov%V=2Q#(Dc^jgpqG_Yvf&Pk$nbI(_nzIqj)muuUq-efhJ?Wm#F7&oz#`h z-DPIVFfX@bh!DYfdB>TTi^rbg`?U`ouZN3RQaA9hRE|80ENZ{K_5w?;8&S{CLS1og zYCasl?L5qj2oYp?u@={O$E47{ur7To+cx`9t$}j^JS>sxIC?(!Ol~l+sL8iu>-kxz zbs6Eb7k@`LZ9;?yvgG+7%dgz?Pl08_q@o!jme{R4Ebn)4SfX0?i3XOZilE-rb2 zdM=H=W)6D(HIN9YNzW@THpL^LZcG8b)KWf_h?Ah_I-#ABu2AFW0?@OJsa8*LA-i?8-5=rzR@PL1)yi^c|=An^}KiIF-5p~ zp5;b_(R!}4f%69YM#~RTuq*E@B?2Vq+4=(KWxJ~umEAzkD_0Xq((0KS;E||D7eUV@ z>xtN!^?bZ67xlc&af>zR`Rf)UP%1sYNL`P5-g&#@4QLUQb`qIFEy7!G9kz&zjl&H< z&!-}YnA++2{O1kbpmna@_T{Ch-9&EZX`QQYavpWbS<}5eG?}CbqMpBXQonKT`rgw6 z<|X$C5h64%GWR(1Qh$75Hn5EB)}l*swbDX%@~})fziJnHJr7uB zn+PnPgNYD9mfP!bjdiqVz6O@fVT+-rU^%pp z2oYqtv;kSH&vna#MwpYJQSW2*9?RS(QO^rkajxN3Y;(n{=aNfAl8B1Zv-`(fv_!C< z-3`!l;te9CCOtRPY=U|Y4Sm`i=;SkrI0e*u8!(!0$m>)zK zt>@To6L42Pb#7%1>tZ96Ss$-n+w_fxe7vXAnxn7%uMEN!2J2LMaXz5fkK zKX$)h3~q0EIlThKFL}3+48iG_HC?EFS^j-ZB}(C8|L}AL*e}1=;;qt5?a>O>=qj}( zD86!ggJcr+D(!R2ebg$A;eI+3N|hG)1<4RXRhn?62x>^T@LpYu;=7_hNXFr*(o(e5 zP^*po+vGQ(RB2soXAt2;R+T0v#V`-0N;BX?G7@){7F&td;|x9ZhaII#OA|vf2y2z5 z)P+{1Eicksh*G7A$|IRUVwJ|tFS!$?N_(k>WEf_Z7JmtCH~!P@GmoOg+e|$q1Mu`X zqchMp4mSOhn~hSXJvK!$iP$O)!-4naa5AD)X{EME#uBd5BAcI~R%z?6ryfP=amKqL z8Aw!>rZJ}vwMx5n-xl?X>o0ytX5uO$8fv>yi-=$4=bocfX^LS;#*kK}wFx;mqZFO` zS@T=K#m$*@Sc=Y+bhHzlBt0*o7M=Hsko03OIxz+ws}=vtqZFN+tC0-B6)p_R(8A^9 zg)Y=<%;R=n_y+dNflXNa5+u9}wQ#yKYvpp30`uTeB>mX^g3+_FIHZF*Cx74^k|8+# zvY-bwjxF<}xKZN8jy=NbJ*Z0CvJH!0vHJxhUba8Utwf2J zH8Mzs;PlI^UR1wq-#YtI!a0E<<#{E?3XvYS+p=q(u3t_o4|{=2uh>4gEbJz e0G#glhUO0L?-dd#Wq@iVk};%}0hhz`rvm`-lNogY literal 0 HcmV?d00001 diff --git a/logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/trial_1/flags_0.json b/logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/trial_1/flags_0.json new file mode 100644 index 00000000..06c6e3fa --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/trial_1/flags_0.json @@ -0,0 +1,62 @@ +{ + "logtostderr": false, + "alsologtostderr": false, + "log_dir": "", + "v": 0, + "verbosity": 0, + "logger_levels": {}, + "stderrthreshold": "fatal", + "showprefixforinfo": true, + "run_with_pdb": false, + "pdb_post_mortem": false, + "pdb": false, + "run_with_profiling": false, + "profile_file": null, + "use_cprofile_for_profiling": true, + "only_check_args": false, + "op_conversion_fallback_to_while_loop": true, + "delta_threshold": 0.5, + "tt_check_filter": false, + "tt_single_core_summaries": false, + "runtime_oom_exit": true, + "hbm_oom_exit": true, + "test_srcdir": "", + "test_tmpdir": "/tmp/absl_testing", + "test_random_seed": 301, + "test_randomize_ordering_seed": "", + "xml_output_file": "", + "experimental_orbax_use_distributed_process_id": false, + "experimental_orbax_use_distributed_barrier": false, + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "workload": "librispeech_deepspeech", + "tuning_ruleset": "self", + "tuning_search_space": null, + "num_tuning_trials": 1, + "data_dir": "/data/librispeech", + "imagenet_v2_data_dir": null, + "librispeech_tokenizer_vocab_path": "/data/librispeech/spm_model.vocab", + "framework": "pytorch", + "torch_compile": true, + "experiment_dir": "/experiment_runs", + "experiment_name": "submissions_a100/ademamix_golden/study_2", + "save_checkpoints": false, + "save_intermediate_checkpoints": true, + "resume_last_run": null, + "append_timestamp": false, + "use_wandb": false, + "profile": false, + "max_global_steps": null, + "overwrite": true, + "hparam_start_index": null, + "hparam_end_index": null, + "rng_seed": -618110163, + "set_pytorch_max_split_size": false, + "pytorch_eval_num_workers": 0, + "capture_jax_trace": false, + "skip_evals": false, + "?": false, + "help": false, + "helpshort": false, + "helpfull": false, + "helpxml": false +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/trial_1/measurements.csv b/logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/trial_1/measurements.csv new file mode 100644 index 00000000..6aebd20b --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/trial_1/measurements.csv @@ -0,0 +1,288 @@ +global_step,grad_norm,loss,train/ctc_loss,train/wer,validation/ctc_loss,validation/wer,validation/num_examples,test/ctc_loss,test/wer,test/num_examples,score,total_duration,accumulated_submission_time,accumulated_eval_time,accumulated_logging_time,preemption_count +0,0.4999999701976776,33.44583511352539,,,,,,,,,,,,,, +1,,,32.54331867872724,3.611568332605732,31.47319858850713,3.283097571573408,5348.0,31.595735208723145,3.6465785144110656,2472.0,54.36974382400513,121.9671106338501,54.36974382400513,67.00936770439148,0.0,0.0 +1,0.4999999701976776,32.872276306152344,,,,,,,,,,,,,, +2,0.4999999701976776,33.36300277709961,,,,,,,,,,,,,, +3,0.5,33.39585876464844,,,,,,,,,,,,,, +4,0.4999999701976776,32.838592529296875,,,,,,,,,,,,,, +5,0.5,33.02530670166016,,,,,,,,,,,,,, +6,0.4999999403953552,32.966148376464844,,,,,,,,,,,,,, +7,0.4999999403953552,31.83272171020508,,,,,,,,,,,,,, +8,0.5,31.989192962646484,,,,,,,,,,,,,, +9,0.4999999403953552,31.371456146240234,,,,,,,,,,,,,, +10,0.5,31.15302848815918,,,,,,,,,,,,,, +11,0.5,30.91205406188965,,,,,,,,,,,,,, +12,0.4999999701976776,30.91806983947754,,,,,,,,,,,,,, +13,0.5,29.997879028320312,,,,,,,,,,,,,, +14,0.5,29.851634979248047,,,,,,,,,,,,,, +15,0.4999999701976776,28.57587242126465,,,,,,,,,,,,,, +16,0.4999999701976776,28.98032760620117,,,,,,,,,,,,,, +17,0.5,28.56360626220703,,,,,,,,,,,,,, +18,0.5,28.12721061706543,,,,,,,,,,,,,, +19,0.4999999403953552,27.41386985778809,,,,,,,,,,,,,, +20,0.4999999403953552,26.636213302612305,,,,,,,,,,,,,, +21,0.4999999701976776,26.506237030029297,,,,,,,,,,,,,, +22,0.4999999403953552,26.75180435180664,,,,,,,,,,,,,, +23,0.5,25.65583229064941,,,,,,,,,,,,,, +24,0.4999999701976776,25.428096771240234,,,,,,,,,,,,,, +25,0.4999999105930328,25.186691284179688,,,,,,,,,,,,,, +26,0.4999999701976776,24.32077980041504,,,,,,,,,,,,,, +27,0.4999999701976776,24.104642868041992,,,,,,,,,,,,,, +28,0.4999999701976776,23.027162551879883,,,,,,,,,,,,,, +29,0.4999999701976776,22.1892032623291,,,,,,,,,,,,,, +30,0.5,21.79735374450684,,,,,,,,,,,,,, +31,0.4999999105930328,21.096790313720703,,,,,,,,,,,,,, +32,0.4999999701976776,20.05625343322754,,,,,,,,,,,,,, +33,0.4999999403953552,19.82076072692871,,,,,,,,,,,,,, +34,0.4999999403953552,19.02323341369629,,,,,,,,,,,,,, +35,0.4999999701976776,18.683698654174805,,,,,,,,,,,,,, +36,0.4999999105930328,17.711366653442383,,,,,,,,,,,,,, +37,0.4999999403953552,16.565828323364258,,,,,,,,,,,,,, +38,0.4999999403953552,16.46105194091797,,,,,,,,,,,,,, +39,0.4999999105930328,15.934101104736328,,,,,,,,,,,,,, +40,0.5,14.955791473388672,,,,,,,,,,,,,, +41,0.4999999403953552,14.191181182861328,,,,,,,,,,,,,, +42,0.5,13.588744163513184,,,,,,,,,,,,,, +43,0.4999999105930328,13.016983032226562,,,,,,,,,,,,,, +44,0.4999999701976776,12.464058876037598,,,,,,,,,,,,,, +45,0.5,11.812773704528809,,,,,,,,,,,,,, +46,0.4999999403953552,11.204166412353516,,,,,,,,,,,,,, +47,0.4999999403953552,10.919561386108398,,,,,,,,,,,,,, +48,0.4999999105930328,10.98756217956543,,,,,,,,,,,,,, +49,0.5,10.570812225341797,,,,,,,,,,,,,, +50,0.4999999105930328,10.064844131469728,,,,,,,,,,,,,, +51,0.4999999105930328,9.55080223083496,,,,,,,,,,,,,, +52,0.4999999403953552,9.38302993774414,,,,,,,,,,,,,, +53,0.4999999105930328,9.175274848937988,,,,,,,,,,,,,, +54,0.4999999105930328,9.060112953186035,,,,,,,,,,,,,, +55,0.4999998807907104,8.94384479522705,,,,,,,,,,,,,, +56,0.4999999105930328,8.52011775970459,,,,,,,,,,,,,, +57,0.4999999105930328,8.46748161315918,,,,,,,,,,,,,, +58,0.4999998807907104,8.039386749267578,,,,,,,,,,,,,, +59,0.4999997913837433,8.031694412231445,,,,,,,,,,,,,, +60,0.4999998807907104,7.803102970123291,,,,,,,,,,,,,, +61,0.4999998211860657,7.608560562133789,,,,,,,,,,,,,, +62,0.499999850988388,7.653901100158691,,,,,,,,,,,,,, +63,0.4999998807907104,7.518303871154785,,,,,,,,,,,,,, +64,0.4999998211860657,7.343469142913818,,,,,,,,,,,,,, +65,0.499999850988388,7.2786078453063965,,,,,,,,,,,,,, +66,0.4999998807907104,7.357726573944092,,,,,,,,,,,,,, +67,0.4999999105930328,7.29986047744751,,,,,,,,,,,,,, +68,0.4999997615814209,7.157777786254883,,,,,,,,,,,,,, +69,0.4999997317790985,7.146546363830566,,,,,,,,,,,,,, +70,0.4999998807907104,7.037361145019531,,,,,,,,,,,,,, +71,0.4999998211860657,7.120115280151367,,,,,,,,,,,,,, +72,0.5,7.044064998626709,,,,,,,,,,,,,, +73,0.4999999403953552,7.035636901855469,,,,,,,,,,,,,, +74,0.4999997913837433,6.934051036834717,,,,,,,,,,,,,, +75,0.4999998807907104,6.918634414672852,,,,,,,,,,,,,, +76,0.4999998211860657,6.867358684539795,,,,,,,,,,,,,, +77,0.4999999403953552,6.95131254196167,,,,,,,,,,,,,, +78,0.4999999701976776,6.878482818603516,,,,,,,,,,,,,, +79,0.4999997615814209,6.812682628631592,,,,,,,,,,,,,, +80,0.4999997913837433,6.77502965927124,,,,,,,,,,,,,, +81,0.4999998807907104,6.6795172691345215,,,,,,,,,,,,,, +82,0.4999997019767761,6.73084831237793,,,,,,,,,,,,,, +83,0.4999997317790985,6.718360424041748,,,,,,,,,,,,,, +84,0.4999999105930328,6.654674530029297,,,,,,,,,,,,,, +85,0.4999998807907104,6.573612213134766,,,,,,,,,,,,,, +86,0.4999998807907104,6.6361284255981445,,,,,,,,,,,,,, +87,0.4999998807907104,6.577873706817627,,,,,,,,,,,,,, +88,0.4999999403953552,6.53550386428833,,,,,,,,,,,,,, +89,0.4999997615814209,6.538516521453857,,,,,,,,,,,,,, +90,0.4999999105930328,6.465516567230225,,,,,,,,,,,,,, +91,0.4999997913837433,6.515180587768555,,,,,,,,,,,,,, +92,0.4999999403953552,6.502839088439941,,,,,,,,,,,,,, +93,0.4999999403953552,6.4352641105651855,,,,,,,,,,,,,, +94,0.4999997913837433,6.376542091369629,,,,,,,,,,,,,, +95,0.4999997913837433,6.347826957702637,,,,,,,,,,,,,, +96,0.4999998807907104,6.392574787139893,,,,,,,,,,,,,, +97,0.4999998807907104,6.3185811042785645,,,,,,,,,,,,,, +98,0.4999998211860657,6.305380344390869,,,,,,,,,,,,,, +99,0.4999997615814209,6.26942253112793,,,,,,,,,,,,,, +100,0.4999998807907104,6.305377960205078,,,,,,,,,,,,,, +500,0.4999997913837433,4.647591590881348,,,,,,,,,,,,,, +1000,0.4999997913837433,3.0619702339172363,,,,,,,,,,,,,, +1500,0.4999997913837433,2.6404356956481934,,,,,,,,,,,,,, +1991,,,6.037100156652686,0.9414253453414608,5.9608374303044,0.8964901269733984,5348.0,5.847129653526034,0.8991123839701014,2472.0,1499.6107699871063,1612.9545664787292,1499.6107699871063,110.08822107315063,0.1076393127441406,0.0 +2000,0.4999998807907104,2.429037570953369,,,,,,,,,,,,,, +2500,0.4999998211860657,2.22705626487732,,,,,,,,,,,,,, +3000,0.4999998211860657,2.1049740314483643,,,,,,,,,,,,,, +3500,0.499999850988388,2.0082571506500244,,,,,,,,,,,,,, +3933,,,1.6419855517374973,0.4234104591438011,1.819571591822383,0.4385844638632742,5348.0,1.2938803511516872,0.349074807547783,2472.0,2945.160037517548,3109.749101638794,2945.160037517548,158.6353361606598,0.3192300796508789,0.0 +4000,0.499999850988388,1.932634234428405,,,,,,,,,,,,,, +4500,0.4999998807907104,1.8837023973464968,,,,,,,,,,,,,, +5000,0.4999998807907104,1.8853470087051392,,,,,,,,,,,,,, +5500,0.4999997913837433,1.8478436470031736,,,,,,,,,,,,,, +5867,,,0.5884155970488655,0.1902005486657935,0.8283974446516476,0.234934582146478,5348.0,0.512976744467385,0.1638128897284342,2472.0,4390.693488121033,4607.824376344681,4390.693488121033,208.74199318885803,0.4080448150634765,0.0 +6000,0.499999850988388,1.7736022472381592,,,,,,,,,,,,,, +6500,0.499999850988388,1.729508638381958,,,,,,,,,,,,,, +7000,0.4999997913837433,1.6954365968704224,,,,,,,,,,,,,, +7500,0.499999850988388,1.7144075632095337,,,,,,,,,,,,,, +7769,,,0.4873721933060454,0.1617162783427921,0.7424400288200723,0.2120600589002076,5348.0,0.4462445157752113,0.1446184469766213,2472.0,5836.115516662598,6103.235170602799,5836.115516662598,256.05564308166504,0.4997012615203857,0.0 +8000,0.4999998211860657,1.5607727766036987,,,,,,,,,,,,,, +8500,0.4999997913837433,1.5532077550888062,,,,,,,,,,,,,, +9000,0.499999850988388,1.6583079099655151,,,,,,,,,,,,,, +9500,0.499999850988388,1.5993014574050903,,,,,,,,,,,,,, +9590,,,0.4252956714167804,0.1424644687697059,0.6756669056566456,0.1934147636749867,5348.0,0.3939028324408026,0.128998842239961,2472.0,7281.850630283356,7597.046008348465,7281.850630283356,301.57932806015015,0.6819305419921875,0.0 +10000,0.4999998807907104,1.6857105493545532,,,,,,,,,,,,,, +10500,0.499999850988388,1.5856595039367676,,,,,,,,,,,,,, +11000,0.4999998807907104,1.5179767608642578,,,,,,,,,,,,,, +11351,,,0.3833944731500665,0.1296748427571264,0.6379087945172794,0.1844542075025346,5348.0,0.3634641247015936,0.1171775028944001,2472.0,8727.689821004868,9090.325487852097,8727.689821004868,346.61975502967834,0.7487177848815918,0.0 +11500,0.4999998211860657,1.502875804901123,,,,,,,,,,,,,, +12000,0.499999850988388,1.473860263824463,,,,,,,,,,,,,, +12500,0.4999998807907104,1.5132384300231934,,,,,,,,,,,,,, +13000,0.4999999105930328,1.5206013917922974,,,,,,,,,,,,,, +13269,,,0.3551276908888776,0.1223880436130019,0.6118201343366988,0.1745763530150147,5348.0,0.3382709166720433,0.111023094266041,2472.0,10173.496448755264,10584.515745162964,10173.496448755264,392.4531898498535,0.8087592124938965,0.0 +13500,0.4999998211860657,1.375339388847351,,,,,,,,,,,,,, +14000,0.499999850988388,1.3986804485321045,,,,,,,,,,,,,, +14500,0.4999999105930328,1.4587703943252563,,,,,,,,,,,,,, +15000,0.4999998807907104,1.4592647552490234,,,,,,,,,,,,,, +15193,,,0.3283039978018091,0.1129184385122425,0.5931179421432339,0.1685414956790421,5348.0,0.3243494287937931,0.1052952288099445,2472.0,11619.135604143145,12081.224436283112,11619.135604143145,440.9172077178955,1.01652193069458,0.0 +15500,0.4999999701976776,1.3528969287872314,,,,,,,,,,,,,, +16000,0.4999999403953552,1.387062907218933,,,,,,,,,,,,,, +16500,0.4999999403953552,1.4207258224487305,,,,,,,,,,,,,, +17000,0.4999999105930328,1.3711750507354736,,,,,,,,,,,,,, +17119,,,0.3036698365674533,0.1057609908322149,0.5600870449379647,0.1598416453435041,5348.0,0.3067811582602103,0.0993033128186379,2472.0,13065.391492843628,13577.54152393341,13065.391492843628,488.4095978736877,1.1076455116271973,0.0 +17500,0.4999999105930328,1.4579237699508667,,,,,,,,,,,,,, +18000,0.4999999403953552,1.3739477396011353,,,,,,,,,,,,,, +18500,0.4999998807907104,1.3552361726760864,,,,,,,,,,,,,, +18990,,,0.2967907781258949,0.1031631822616025,0.5587584725675607,0.1591657413218751,5348.0,0.2995533229966449,0.0980643064611134,2472.0,14511.024166345596,15071.766645908356,14511.024166345596,534.4881474971771,1.189727783203125,0.0 +19000,0.4999999701976776,1.3325650691986084,,,,,,,,,,,,,, +19500,0.4999999105930328,1.4557597637176514,,,,,,,,,,,,,, +20000,0.4999998211860657,1.3754034042358398,,,,,,,,,,,,,, +20500,0.4999999403953552,1.391093373298645,,,,,,,,,,,,,, +20770,,,0.2835062776879411,0.0986736085285732,0.5481313652175005,0.1545985612900111,5348.0,0.2936532447657913,0.0955456705868015,2472.0,15956.933192014694,16565.886620759964,15956.933192014694,580.2749526500702,1.2638778686523438,0.0 +21000,0.4999999403953552,1.3774032592773438,,,,,,,,,,,,,, +21500,0.4999998807907104,1.3304823637008667,,,,,,,,,,,,,, +22000,0.4999998807907104,1.3079915046691897,,,,,,,,,,,,,, +22500,0.4999999105930328,1.3382693529129028,,,,,,,,,,,,,, +22608,,,0.2634278618592821,0.0929120787319244,0.5222968526314798,0.148640950127939,5348.0,0.2798856982708562,0.0906099567363353,2472.0,17403.099194288254,18061.93473124504,17403.099194288254,627.3567457199097,1.4887628555297852,0.0 +23000,0.499999850988388,1.302251219749451,,,,,,,,,,,,,, +23500,0.4999999105930328,1.293820858001709,,,,,,,,,,,,,, +24000,0.4999999403953552,1.2915489673614502,,,,,,,,,,,,,, +24500,0.4999999403953552,1.322284698486328,,,,,,,,,,,,,, +24531,,,0.2469450620715212,0.0870697042702152,0.5107892791214587,0.1445758702264278,5348.0,0.2715927237241112,0.0873397924156561,2472.0,18848.515809059143,19555.506080389023,18848.515809059143,673.05593085289,1.5548083782196045,0.0 +25000,0.499999850988388,1.3020564317703247,,,,,,,,,,,,,, +25500,0.4999999403953552,1.2602529525756836,,,,,,,,,,,,,, +26000,0.4999999403953552,1.309916377067566,,,,,,,,,,,,,, +26461,,,0.2298987653073256,0.0818148010412792,0.4952058767204139,0.1402500844880027,5348.0,0.2620092929382541,0.0849024028598704,2472.0,20293.84050965309,21050.80857372284,20293.84050965309,720.5358953475952,1.6216716766357422,0.0 +26500,0.4999999701976776,1.246646523475647,,,,,,,,,,,,,, +27000,0.4999998807907104,1.272379755973816,,,,,,,,,,,,,, +27500,0.4999998211860657,1.2220838069915771,,,,,,,,,,,,,, +28000,0.4999999403953552,1.2066681385040283,,,,,,,,,,,,,, +28356,,,0.2168352156922195,0.0781552325362049,0.48093368165436,0.1355960025105006,5348.0,0.2542665807068198,0.0817134848577173,2472.0,21739.44190144539,22547.26208972931,21739.44190144539,768.8759837150574,1.793954372406006,0.0 +28500,0.4999998807907104,1.1740680932998655,,,,,,,,,,,,,, +29000,0.4999999105930328,1.179255127906799,,,,,,,,,,,,,, +29500,0.4999999105930328,1.2885137796401978,,,,,,,,,,,,,, +30000,0.4999999105930328,1.2729668617248535,,,,,,,,,,,,,, +30172,,,0.2034263037546111,0.073875854932333,0.4689482753478501,0.13204267851108,5348.0,0.2475520094038325,0.0791542258241423,2472.0,23185.21777319908,24041.661779642105,23185.21777319908,815.013129234314,1.8652231693267824,0.0 +30500,0.4999999105930328,1.2352018356323242,,,,,,,,,,,,,, +31000,0.4999999403953552,1.192611813545227,,,,,,,,,,,,,, +31500,0.4999999403953552,1.2040177583694458,,,,,,,,,,,,,, +31942,,,0.1944966005946064,0.0704857686441271,0.458238463337603,0.130237049196157,5348.0,0.2399674979030905,0.0764121625738833,2472.0,24630.957630872726,25535.483829259872,24630.957630872726,860.614027261734,1.936441421508789,0.0 +32000,0.4999999701976776,1.225269436836243,,,,,,,,,,,,,, +32500,0.4999998807907104,1.125121831893921,,,,,,,,,,,,,, +33000,0.4999999105930328,1.1744414567947388,,,,,,,,,,,,,, +33500,0.4999999701976776,1.2388126850128174,,,,,,,,,,,,,, +33864,,,0.1845796065154041,0.0668800965824265,0.4528886589248041,0.1274272196205281,5348.0,0.235614999354797,0.0753153372737797,2472.0,26076.77673649788,27031.1519382,26076.77673649788,907.7446002960204,2.1716549396514893,0.0 +34000,0.4999998807907104,1.143073320388794,,,,,,,,,,,,,, +34500,0.4999998807907104,1.1401106119155884,,,,,,,,,,,,,, +35000,0.4999999105930328,1.1348857879638672,,,,,,,,,,,,,, +35500,0.4999999403953552,1.1864606142044067,,,,,,,,,,,,,, +35799,,,0.1793803138738693,0.0649182660436237,0.448043519721971,0.1254381306425916,5348.0,0.2331130486241047,0.0738935266995714,2472.0,27523.128769874573,28527.29335784912,27523.128769874573,954.985598564148,2.241617202758789,0.0 +36000,0.5,1.1818891763687134,,,,,,,,,,,,,, +36500,0.4999999105930328,1.1434181928634644,,,,,,,,,,,,,, +37000,0.5,1.148813247680664,,,,,,,,,,,,,, +37500,0.4999999105930328,1.2023475170135498,,,,,,,,,,,,,, +37709,,,0.1775322359223137,0.0643361844551878,0.4462745425833835,0.1246173900448993,5348.0,0.231312252000129,0.0732638677309934,2472.0,28968.81560564041,30022.68916273117,28968.81560564041,1002.081166267395,2.312926769256592,0.0 +38000,0.4999999403953552,1.131960391998291,,,,,,,,,,,,,, +38500,0.4999999403953552,1.193366289138794,,,,,,,,,,,,,, +39000,0.4999999701976776,1.190051555633545,,,,,,,,,,,,,, +39500,0.4999999403953552,1.1563535928726196,,,,,,,,,,,,,, +39565,,,0.1771153639648289,0.064223001924103,0.446010584234353,0.1244435861536233,5348.0,0.2313203422398219,0.0732841793106249,2472.0,30414.163591861725,31518.050532341003,30414.163591861725,1049.497998714447,2.476557493209839,0.0 +40000,0.4999999403953552,1.257368803024292,,,,,,,,,,,,,, +40500,0.499999850988388,1.1653704643249512,,,,,,,,,,,,,, +41000,0.4999999701976776,1.1071758270263672,,,,,,,,,,,,,, +41333,,,0.176319414615864,0.0638618957534992,0.4440256723898684,0.1243953072949355,5348.0,0.2310354196238467,0.0731419982532041,2472.0,31859.80902147293,33012.86257123947,31859.80902147293,1096.234617948532,2.5408711433410645,0.0 +41500,0.4999999105930328,1.2394551038742063,,,,,,,,,,,,,, +42000,0.5,1.189550518989563,,,,,,,,,,,,,, +42500,0.4999999701976776,1.1515016555786133,,,,,,,,,,,,,, +43000,0.4999999403953552,1.124111533164978,,,,,,,,,,,,,, +43191,,,0.1763380618651776,0.0641529365477172,0.4456140825924251,0.1243277168927726,5348.0,0.2310111741080069,0.0728170129790993,2472.0,33305.35944676399,34511.06227684021,33305.35944676399,1146.3031606674194,2.6046295166015625,0.0 +43500,0.5,1.1478029489517212,,,,,,,,,,,,,, +44000,0.4999998807907104,1.2172892093658447,,,,,,,,,,,,,, +44500,0.4999999701976776,1.2567417621612549,,,,,,,,,,,,,, +45000,0.4999999403953552,1.1313999891281128,,,,,,,,,,,,,, +45127,,,0.1780934036375427,0.0647026802701289,0.4487201674578059,0.1250325882296142,5348.0,0.2346047023195044,0.0750715983182012,2472.0,34752.82623553276,36009.81967043877,34752.82623553276,1194.8423812389374,2.794597625732422,0.0 +45500,0.4999999105930328,1.1902583837509155,,,,,,,,,,,,,, +46000,0.4999999403953552,1.1709177494049072,,,,,,,,,,,,,, +46500,0.4999999403953552,1.2122529745101929,,,,,,,,,,,,,, +47000,0.4999999105930328,1.3184365034103394,,,,,,,,,,,,,, +47058,,,0.1830817994424512,0.0668639276494144,0.4555611084425859,0.1279969101530439,5348.0,0.2380288143589909,0.0745638088274125,2472.0,36198.51755785942,37505.41343545914,36198.51755785942,1242.190561056137,2.874882936477661,0.0 +47500,0.4999999403953552,1.1512194871902466,,,,,,,,,,,,,, +48000,0.4999998807907104,1.2348545789718628,,,,,,,,,,,,,, +48500,0.4999998807907104,1.2376641035079956,,,,,,,,,,,,,, +48961,,,0.1920238463708794,0.0703995343347292,0.472720902871961,0.1316178245546275,5348.0,0.2460183419091554,0.0781589584221965,2472.0,37644.09406256676,39000.54671049118,37644.09406256676,1289.2322795391085,2.995460033416748,0.0 +49000,0.4999998807907104,1.241675853729248,,,,,,,,,,,,,, +49500,0.4999998211860657,1.1930099725723269,,,,,,,,,,,,,, +50000,0.4999999403953552,1.2351584434509275,,,,,,,,,,,,,, +50500,0.4999999105930328,1.1798133850097656,,,,,,,,,,,,,, +50783,,,0.2052840014233496,0.0759346990692084,0.4910192353576452,0.1385313571187177,5348.0,0.2585415288889606,0.0828103101578209,2472.0,39089.643002033234,40494.20410013199,39089.643002033234,1334.7426974773407,3.2001163959503174,0.0 +51000,0.4999999403953552,1.2648168802261353,,,,,,,,,,,,,, +51500,0.4999998807907104,1.2556246519088743,,,,,,,,,,,,,, +52000,0.5,1.2324508428573608,,,,,,,,,,,,,, +52500,0.4999999701976776,1.3101752996444702,,,,,,,,,,,,,, +52559,,,0.2161155397779911,0.0791954338933173,0.4994076756988396,0.14022111717279,5348.0,0.2638593114878379,0.083480592285662,2472.0,40535.483557224274,41988.20076799393,40535.483557224274,1380.3914070129397,3.269481897354126,0.0 +53000,0.4999999105930328,1.4210878610610962,,,,,,,,,,,,,, +53500,0.4999999403953552,1.3073279857635498,,,,,,,,,,,,,, +54000,0.4999999403953552,1.247300386428833,,,,,,,,,,,,,, +54474,,,0.2324301326915626,0.0842940374364695,0.5142982916729455,0.1447400183459663,5348.0,0.2708677525566165,0.0870960534600776,2472.0,41983.54762744904,43484.77414178848,41983.54762744904,1426.375376701355,3.331726312637329,0.0 +54500,0.4999999403953552,1.30730402469635,,,,,,,,,,,,,, +55000,0.4999999701976776,1.2625324726104736,,,,,,,,,,,,,, +55500,0.4999999105930328,1.3544447422027588,,,,,,,,,,,,,, +56000,0.4999998807907104,1.2384603023529053,,,,,,,,,,,,,, +56411,,,0.2557255609176815,0.0928258444225265,0.5407247361299478,0.1506107275624004,5348.0,0.2903184782082715,0.09408323685333,2472.0,43428.78943896294,44979.52585673332,43428.78943896294,1473.2940378189087,3.570307731628418,0.0 +56500,0.4999999403953552,1.3465713262557983,,,,,,,,,,,,,, +57000,0.4999999105930328,1.2776365280151367,,,,,,,,,,,,,, +57500,0.4999998807907104,1.4377851486206057,,,,,,,,,,,,,, +58000,0.4999999105930328,1.3499034643173218,,,,,,,,,,,,,, +58326,,,0.2634646299711961,0.0942487105275922,0.5477415839423849,0.1523970453338483,5348.0,0.2869752937689528,0.0940019905348038,2472.0,44874.081411361694,46473.932918787,44874.081411361694,1519.8126411437988,3.655649900436402,0.0 +58500,0.5,1.3514275550842283,,,,,,,,,,,,,, +59000,0.499999850988388,1.3424222469329834,,,,,,,,,,,,,, +59500,0.4999999403953552,1.4068506956100464,,,,,,,,,,,,,, +60000,0.4999999701976776,1.354491114616394,,,,,,,,,,,,,, +60188,,,0.2630659183553152,0.0951972879309694,0.535008598145218,0.1505238256167624,5348.0,0.2884309068326989,0.0931895273495419,2472.0,46319.49430727959,47967.867362737656,46319.49430727959,1565.8220901489258,3.750631093978882,0.0 +60500,0.4999999701976776,1.4678750038146973,,,,,,,,,,,,,, +61000,0.4999998807907104,1.3515044450759888,,,,,,,,,,,,,, +61500,0.5,1.313547968864441,,,,,,,,,,,,,, +61973,,,0.2825832322670844,0.0995575101999019,0.5602312139749096,0.1548689228986626,5348.0,0.3050043047132073,0.0973534011740093,2472.0,47764.89419078827,49461.83552980423,47764.89419078827,1611.953557252884,3.8260538578033447,0.0 +62000,0.4999999403953552,1.3640669584274292,,,,,,,,,,,,,, +62500,0.4999999105930328,1.2697386741638184,,,,,,,,,,,,,, +63000,0.4999999701976776,1.2948966026306152,,,,,,,,,,,,,, +63500,0.4999999403953552,1.3677994012832642,,,,,,,,,,,,,, +63809,,,0.2819004034227769,0.1000910849893015,0.5507110049979907,0.155535171148554,5348.0,0.3006939207771469,0.0977596327666402,2472.0,49210.93877458573,50956.652635097504,49210.93877458573,1657.9822750091553,4.041477918624878,0.0 +64000,0.4999999403953552,1.380255103111267,,,,,,,,,,,,,, +64500,0.4999999403953552,1.386572241783142,,,,,,,,,,,,,, +65000,0.5,1.3665876388549805,,,,,,,,,,,,,, +65500,0.4999999105930328,1.3203010559082031,,,,,,,,,,,,,, +65748,,,0.2963697213940404,0.1038853946028101,0.568032922144992,0.1609424033215854,5348.0,0.3072860547454674,0.0972924664351146,2472.0,50657.33596086502,52451.73114442825,50657.33596086502,1704.2230117321014,4.118767023086548,0.0 +66000,0.4999999403953552,1.3218470811843872,,,,,,,,,,,,,, +66500,0.4999999701976776,1.4319453239440918,,,,,,,,,,,,,, +67000,0.4999999403953552,1.363422393798828,,,,,,,,,,,,,, +67500,0.4999999403953552,1.363980412483215,,,,,,,,,,,,,, +67681,,,0.3175066219448515,0.1108110875763308,0.5771099793424754,0.1646309081253319,5348.0,0.3143680892718885,0.1023500497633701,2472.0,52103.00220417976,53947.36892056465,52103.00220417976,1751.6461789608002,4.180283546447754,0.0 +68000,0.4999998807907104,1.3576505184173584,,,,,,,,,,,,,, +68500,0.4999999701976776,1.2784221172332764,,,,,,,,,,,,,, +69000,0.4999999701976776,1.3623377084732056,,,,,,,,,,,,,, +69500,0.4999999105930328,1.383233904838562,,,,,,,,,,,,,, +69583,,,0.3075975499856823,0.1068982057874,0.5708020566480811,0.1611451745280741,5348.0,0.3124170813439577,0.0995470517742164,2472.0,53548.67119860649,55441.55384373665,53548.67119860649,1797.4855284690857,4.375433206558228,0.0 +70000,0.4999998807907104,1.5175552368164062,,,,,,,,,,,,,, +70500,0.4999999701976776,1.4747676849365234,,,,,,,,,,,,,, +71000,0.5,1.4289627075195312,,,,,,,,,,,,,, +71419,,,0.311029117607425,0.109140297831746,0.579972463174352,0.1622748998213682,5348.0,0.313425916591393,0.0993845591371641,2472.0,54994.37272357941,56936.35423111916,54994.37272357941,1844.1046738624573,4.456562280654907,0.0 +71500,0.4999999403953552,1.41721773147583,,,,,,,,,,,,,, +72000,0.4999999105930328,1.4537146091461182,,,,,,,,,,,,,, +72500,0.4999999403953552,1.3436925411224363,,,,,,,,,,,,,, +73000,0.5,1.450686812400818,,,,,,,,,,,,,, +73196,,,,,,,,,,,56439.75518965721,,,,,0.0 diff --git a/logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/trial_1/meta_data_0.json b/logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/trial_1/meta_data_0.json new file mode 100644 index 00000000..629e4f0f --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/librispeech_deepspeech_pytorch/trial_1/meta_data_0.json @@ -0,0 +1,77 @@ +{ + "workload.attention_temperature": 1.0, + "workload.enable_decoder_layer_norm": true, + "workload.enable_residual_connections": true, + "workload.eval_batch_size": 256, + "workload.eval_num_workers": 0, + "workload.eval_period_time_sec": 1447, + "workload.freq_mask_count": 2, + "workload.layernorm_everywhere": false, + "workload.max_allowed_runtime_sec": 36949, + "workload.num_eval_train_examples": 5376, + "workload.num_test_examples": 2472, + "workload.num_train_examples": 263840, + "workload.num_validation_examples": 5348, + "workload.requires_sync_before_eval": false, + "workload.step_hint": 38400, + "workload.target_metric_name": "wer", + "workload.test_target_value": 0.074143, + "workload.time_mask_count": 10, + "workload.use_gelu": false, + "workload.use_post_layer_norm": true, + "workload.use_specaug": true, + "workload.use_tanh": false, + "workload.validation_target_value": 0.119936, + "cpu.util.avg_percent_since_last": 5.4, + "cpu.freq.current": 2200.2180000000003, + "mem.total": 359053524992, + "mem.available": 348464537600, + "mem.used": 7433154560, + "mem.percent_used": 2.9, + "mem.read_bytes_since_boot": 15630197489152, + "mem.write_bytes_since_boot": 45558990848, + "net.bytes_sent_since_boot": 31338216, + "net.bytes_recv_since_boot": 31341309, + "gpu.count": 4, + "gpu.0.compute.util": 0.0, + "gpu.0.mem.util": 0.0309326171875, + "gpu.0.mem.total": 40960.0, + "gpu.0.mem.used": 1267.0, + "gpu.0.mem.free": 39060.0, + "gpu.0.temp.current": 35.0, + "gpu.1.compute.util": 0.0, + "gpu.1.mem.util": 0.0309326171875, + "gpu.1.mem.total": 40960.0, + "gpu.1.mem.used": 1267.0, + "gpu.1.mem.free": 39060.0, + "gpu.1.temp.current": 33.0, + "gpu.2.compute.util": 0.0, + "gpu.2.mem.util": 0.0309326171875, + "gpu.2.mem.total": 40960.0, + "gpu.2.mem.used": 1267.0, + "gpu.2.mem.free": 39060.0, + "gpu.2.temp.current": 32.0, + "gpu.3.compute.util": 0.0, + "gpu.3.mem.util": 0.0309326171875, + "gpu.3.mem.total": 40960.0, + "gpu.3.mem.used": 1267.0, + "gpu.3.mem.free": 39060.0, + "gpu.3.temp.current": 34.0, + "gpu.avg.compute.util": 0.0, + "gpu.avg.mem.util": 0.0309326171875, + "gpu.avg.mem.total": 40960.0, + "gpu.avg.mem.used": 1267.0, + "gpu.avg.mem.free": 39060.0, + "gpu.avg.temp.current": 33.5, + "os_platform": "Linux-6.1.0-44-cloud-amd64-x86_64-with-glibc2.31", + "python_version": "3.11.10", + "python_compiler": "GCC 9.4.0", + "git_branch": "main", + "git_commit_hash": "b21be29be0a1573fb4f78f849aea019cdb520862", + "cpu_model_name": "Intel(R) Xeon(R) CPU @ 2.20GHz", + "cpu_count": 24, + "gpu_model_name": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_driver": "550.90.12", + "rng_seed": -618110163 +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_2/ogbg_pytorch/ogbg_pytorch_09-15-2026-07-17-58.log b/logs/self_tuning/ademamix_golden/study_2/ogbg_pytorch/ogbg_pytorch_09-15-2026-07-17-58.log new file mode 100644 index 00000000..1ee6ff9b --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/ogbg_pytorch/ogbg_pytorch_09-15-2026-07-17-58.log @@ -0,0 +1,586 @@ +torchrun --redirects 1:0,2:0,3:0 --standalone --nnodes=1 --nproc_per_node=4 submission_runner.py --framework=pytorch --workload=ogbg --submission_path=submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py --data_dir=/data/ogbg --experiment_dir=/experiment_runs --experiment_name=submissions_a100/ademamix_golden/study_2 --overwrite=True --save_checkpoints=False --rng_seed=-777571328 --torch_compile=true --tuning_ruleset=self 2>&1 | tee -a /logs/ogbg_pytorch_09-15-2026-07-17-58.log +W0915 07:18:27.131000 9 site-packages/torch/distributed/run.py:803] +W0915 07:18:27.131000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0915 07:18:27.131000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0915 07:18:27.131000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-15 07:18:42.778574: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-15 07:18:42.778574: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-15 07:18:42.778574: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-15 07:18:42.778575: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789456723.373839 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789456723.373843 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789456723.373839 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789456723.373842 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789456723.515608 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789456723.515614 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789456723.515621 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789456723.515629 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789456725.013904 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789456725.013905 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789456725.013905 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789456725.013907 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789456725.013944 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789456725.013944 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789456725.013947 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789456725.013947 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789456725.013948 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789456725.013949 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789456725.013949 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789456725.013950 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789456725.013950 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789456725.013952 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789456725.013952 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789456725.013954 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789456755.983167 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789456755.995524 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789456756.005691 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789456756.017897 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W915 07:19:27.325000994 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0915 07:19:30.120566 140312702207168 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/ogbg_pytorch. +I0915 07:19:30.120564 139730382337216 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/ogbg_pytorch. +I0915 07:19:30.120564 140042820605120 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/ogbg_pytorch. +I0915 07:19:30.120594 140198333088960 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/ogbg_pytorch. +I0915 07:19:30.329825 140312702207168 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/ogbg_pytorch/trial_1. +I0915 07:19:30.610133 140312702207168 submission_runner.py:242] Initializing dataset. +I0915 07:19:30.610306 140312702207168 submission_runner.py:251] Initializing model. +W0915 07:19:41.672774 140042820605120 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0915 07:19:41.672770 140198333088960 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0915 07:19:41.672772 139730382337216 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +W0915 07:19:41.672803 140312702207168 submission_runner.py:273] These workloads cannot be fully compiled under current PyTorch version. Proceeding without `torch.compile`. +I0915 07:19:45.828102 140312702207168 submission_runner.py:294] Initializing optimizer. +I0915 07:19:45.828739 140312702207168 submission_runner.py:299] Initializing metrics bundle. +I0915 07:19:45.828892 140312702207168 submission_runner.py:321] Initializing checkpoint and logger. +I0915 07:19:45.829876 140312702207168 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_2/ogbg_pytorch/trial_1/meta_data_0.json. +I0915 07:19:45.829930 140198333088960 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0915 07:19:45.829940 139730382337216 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0915 07:19:45.829954 140042820605120 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0915 07:19:45.830069 140312702207168 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0915 07:19:45.830088 139730382337216 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0915 07:19:45.830087 140198333088960 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0915 07:19:45.830101 140042820605120 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0915 07:19:45.830126 140312702207168 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0915 07:19:46.212843 140312702207168 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_2/ogbg_pytorch/trial_1/flags_0.json. +I0915 07:19:46.331853 140312702207168 submission_runner.py:359] Starting training loop. +I0915 07:19:52.514974 140312702207168 dataset_info.py:707] Load dataset info from /data/ogbg/ogbg_molpcba/0.1.3 +I0915 07:19:52.544847 140312702207168 reader.py:262] Creating a tf.data.Dataset reading 8 files located in folders: /data/ogbg/ogbg_molpcba/0.1.3. +WARNING:tensorflow:From /usr/local/lib/python3.11/site-packages/tensorflow_datasets/core/reader.py:102: CounterV2 (from tensorflow.python.data.experimental.ops.counter) is deprecated and will be removed in a future version. +Instructions for updating: +Use `tf.data.Dataset.counter(...)` instead. +W0915 07:19:52.967269 140312702207168 deprecation.py:50] From /usr/local/lib/python3.11/site-packages/tensorflow_datasets/core/reader.py:102: CounterV2 (from tensorflow.python.data.experimental.ops.counter) is deprecated and will be removed in a future version. +Instructions for updating: +Use `tf.data.Dataset.counter(...)` instead. +I0915 07:19:53.474780 140312702207168 logging_logger.py:49] Constructing tf.data.Dataset ogbg_molpcba for split train, from /data/ogbg/ogbg_molpcba/0.1.3 +I0915 07:20:25.493104 140287322027776 logging_writer.py:48] [0] global_step=0, grad_norm=0.5, loss=0.836981 +I0915 07:20:25.818115 140312702207168 submission.py:307] 0) loss = 0.837, grad_norm = 0.500 +I0915 07:20:26.238332 140312702207168 spec.py:333] Evaluating on the training split. +I0915 07:20:26.243132 140312702207168 dataset_info.py:707] Load dataset info from /data/ogbg/ogbg_molpcba/0.1.3 +I0915 07:20:26.246497 140312702207168 reader.py:262] Creating a tf.data.Dataset reading 8 files located in folders: /data/ogbg/ogbg_molpcba/0.1.3. +I0915 07:20:26.310945 140312702207168 logging_logger.py:49] Constructing tf.data.Dataset ogbg_molpcba for split train, from /data/ogbg/ogbg_molpcba/0.1.3 +I0915 07:21:08.704992 140312702207168 spec.py:346] Evaluating on the validation split. +I0915 07:21:08.707292 140312702207168 dataset_info.py:707] Load dataset info from /data/ogbg/ogbg_molpcba/0.1.3 +I0915 07:21:08.710485 140312702207168 reader.py:262] Creating a tf.data.Dataset reading 1 files located in folders: /data/ogbg/ogbg_molpcba/0.1.3. +I0915 07:21:08.769887 140312702207168 logging_logger.py:49] Constructing tf.data.Dataset ogbg_molpcba for split validation, from /data/ogbg/ogbg_molpcba/0.1.3 +I0915 07:21:27.527626 140312702207168 spec.py:363] Evaluating on the test split. +I0915 07:21:27.530133 140312702207168 dataset_info.py:707] Load dataset info from /data/ogbg/ogbg_molpcba/0.1.3 +I0915 07:21:27.533377 140312702207168 reader.py:262] Creating a tf.data.Dataset reading 1 files located in folders: /data/ogbg/ogbg_molpcba/0.1.3. +I0915 07:21:27.596765 140312702207168 logging_logger.py:49] Constructing tf.data.Dataset ogbg_molpcba for split test, from /data/ogbg/ogbg_molpcba/0.1.3 +I0915 07:21:46.384822 140312702207168 submission_runner.py:516] Time since start: 120.05s, Step: 1, {'train/accuracy': 0.4288207246897473, 'train/loss': 0.8422619610364236, 'train/mean_average_precision': 0.02495100353879998, 'validation/accuracy': 0.4298933798576991, 'validation/loss': 0.8421632409340811, 'validation/mean_average_precision': 0.029002724931724184, 'validation/num_examples': 43793, 'test/accuracy': 0.4335950891101118, 'test/loss': 0.8399119195882211, 'test/mean_average_precision': 0.02992468713007351, 'test/num_examples': 43793, 'score': 39.48756432533264, 'total_duration': 120.05303835868835, 'accumulated_submission_time': 39.48756432533264, 'accumulated_eval_time': 80.14644813537598, 'accumulated_logging_time': 0} +I0915 07:21:46.405251 140277943293696 logging_writer.py:48] [1] accumulated_eval_time=80.1464, accumulated_logging_time=0, accumulated_submission_time=39.4876, global_step=1, preemption_count=0, score=39.4876, test/accuracy=0.433595, test/loss=0.839912, test/mean_average_precision=0.0299247, test/num_examples=43793, total_duration=120.053, train/accuracy=0.428821, train/loss=0.842262, train/mean_average_precision=0.024951, validation/accuracy=0.429893, validation/loss=0.842163, validation/mean_average_precision=0.0290027, validation/num_examples=43793 +I0915 07:21:47.148530 140277951686400 logging_writer.py:48] [1] global_step=1, grad_norm=0.5, loss=0.835538 +I0915 07:21:47.151476 140312702207168 submission.py:307] 1) loss = 0.836, grad_norm = 0.500 +I0915 07:21:47.393991 140277943293696 logging_writer.py:48] [2] global_step=2, grad_norm=0.5, loss=0.835827 +I0915 07:21:47.396910 140312702207168 submission.py:307] 2) loss = 0.836, grad_norm = 0.500 +I0915 07:21:47.638208 140277951686400 logging_writer.py:48] [3] global_step=3, grad_norm=0.5, loss=0.833384 +I0915 07:21:47.641033 140312702207168 submission.py:307] 3) loss = 0.833, grad_norm = 0.500 +I0915 07:21:47.884883 140277943293696 logging_writer.py:48] [4] global_step=4, grad_norm=0.5, loss=0.831172 +I0915 07:21:47.887711 140312702207168 submission.py:307] 4) loss = 0.831, grad_norm = 0.500 +I0915 07:21:48.131599 140277951686400 logging_writer.py:48] [5] global_step=5, grad_norm=0.5, loss=0.831098 +I0915 07:21:48.134503 140312702207168 submission.py:307] 5) loss = 0.831, grad_norm = 0.500 +I0915 07:21:48.377891 140277943293696 logging_writer.py:48] [6] global_step=6, grad_norm=0.5, loss=0.828567 +I0915 07:21:48.380978 140312702207168 submission.py:307] 6) loss = 0.829, grad_norm = 0.500 +I0915 07:21:48.624826 140277951686400 logging_writer.py:48] [7] global_step=7, grad_norm=0.5, loss=0.823987 +I0915 07:21:48.627662 140312702207168 submission.py:307] 7) loss = 0.824, grad_norm = 0.500 +I0915 07:21:48.873631 140277943293696 logging_writer.py:48] [8] global_step=8, grad_norm=0.5, loss=0.818522 +I0915 07:21:48.876713 140312702207168 submission.py:307] 8) loss = 0.819, grad_norm = 0.500 +I0915 07:21:49.121839 140277951686400 logging_writer.py:48] [9] global_step=9, grad_norm=0.5, loss=0.813848 +I0915 07:21:49.124961 140312702207168 submission.py:307] 9) loss = 0.814, grad_norm = 0.500 +I0915 07:21:49.363832 140277943293696 logging_writer.py:48] [10] global_step=10, grad_norm=0.5, loss=0.810885 +I0915 07:21:49.366839 140312702207168 submission.py:307] 10) loss = 0.811, grad_norm = 0.500 +I0915 07:21:49.611231 140277951686400 logging_writer.py:48] [11] global_step=11, grad_norm=0.5, loss=0.807572 +I0915 07:21:49.614069 140312702207168 submission.py:307] 11) loss = 0.808, grad_norm = 0.500 +I0915 07:21:49.855910 140277943293696 logging_writer.py:48] [12] global_step=12, grad_norm=0.5, loss=0.795764 +I0915 07:21:49.858695 140312702207168 submission.py:307] 12) loss = 0.796, grad_norm = 0.500 +I0915 07:21:50.102626 140277951686400 logging_writer.py:48] [13] global_step=13, grad_norm=0.5, loss=0.789721 +I0915 07:21:50.105558 140312702207168 submission.py:307] 13) loss = 0.790, grad_norm = 0.500 +I0915 07:21:50.349645 140277943293696 logging_writer.py:48] [14] global_step=14, grad_norm=0.5, loss=0.782695 +I0915 07:21:50.352483 140312702207168 submission.py:307] 14) loss = 0.783, grad_norm = 0.500 +I0915 07:21:50.596763 140277951686400 logging_writer.py:48] [15] global_step=15, grad_norm=0.5, loss=0.774531 +I0915 07:21:50.599876 140312702207168 submission.py:307] 15) loss = 0.775, grad_norm = 0.500 +I0915 07:21:50.841129 140277943293696 logging_writer.py:48] [16] global_step=16, grad_norm=0.5, loss=0.767392 +I0915 07:21:50.843911 140312702207168 submission.py:307] 16) loss = 0.767, grad_norm = 0.500 +I0915 07:21:51.085249 140277951686400 logging_writer.py:48] [17] global_step=17, grad_norm=0.5, loss=0.757034 +I0915 07:21:51.088137 140312702207168 submission.py:307] 17) loss = 0.757, grad_norm = 0.500 +I0915 07:21:51.331360 140277943293696 logging_writer.py:48] [18] global_step=18, grad_norm=0.5, loss=0.750407 +I0915 07:21:51.334232 140312702207168 submission.py:307] 18) loss = 0.750, grad_norm = 0.500 +I0915 07:21:51.578111 140277951686400 logging_writer.py:48] [19] global_step=19, grad_norm=0.5, loss=0.738774 +I0915 07:21:51.580927 140312702207168 submission.py:307] 19) loss = 0.739, grad_norm = 0.500 +I0915 07:21:51.824230 140277943293696 logging_writer.py:48] [20] global_step=20, grad_norm=0.5, loss=0.732378 +I0915 07:21:51.827118 140312702207168 submission.py:307] 20) loss = 0.732, grad_norm = 0.500 +I0915 07:21:52.069875 140277951686400 logging_writer.py:48] [21] global_step=21, grad_norm=0.5, loss=0.72064 +I0915 07:21:52.072788 140312702207168 submission.py:307] 21) loss = 0.721, grad_norm = 0.500 +I0915 07:21:52.312903 140277943293696 logging_writer.py:48] [22] global_step=22, grad_norm=0.5, loss=0.710185 +I0915 07:21:52.315688 140312702207168 submission.py:307] 22) loss = 0.710, grad_norm = 0.500 +I0915 07:21:52.553563 140277951686400 logging_writer.py:48] [23] global_step=23, grad_norm=0.5, loss=0.701896 +I0915 07:21:52.556470 140312702207168 submission.py:307] 23) loss = 0.702, grad_norm = 0.500 +I0915 07:21:52.796157 140277943293696 logging_writer.py:48] [24] global_step=24, grad_norm=0.5, loss=0.692307 +I0915 07:21:52.799107 140312702207168 submission.py:307] 24) loss = 0.692, grad_norm = 0.500 +I0915 07:21:53.039263 140277951686400 logging_writer.py:48] [25] global_step=25, grad_norm=0.5, loss=0.683326 +I0915 07:21:53.042139 140312702207168 submission.py:307] 25) loss = 0.683, grad_norm = 0.500 +I0915 07:21:53.281157 140277943293696 logging_writer.py:48] [26] global_step=26, grad_norm=0.5, loss=0.674155 +I0915 07:21:53.283971 140312702207168 submission.py:307] 26) loss = 0.674, grad_norm = 0.500 +I0915 07:21:53.524739 140277951686400 logging_writer.py:48] [27] global_step=27, grad_norm=0.5, loss=0.665888 +I0915 07:21:53.527595 140312702207168 submission.py:307] 27) loss = 0.666, grad_norm = 0.500 +I0915 07:21:53.768766 140277943293696 logging_writer.py:48] [28] global_step=28, grad_norm=0.5, loss=0.655889 +I0915 07:21:53.771758 140312702207168 submission.py:307] 28) loss = 0.656, grad_norm = 0.500 +I0915 07:21:54.017742 140277951686400 logging_writer.py:48] [29] global_step=29, grad_norm=0.5, loss=0.646958 +I0915 07:21:54.020657 140312702207168 submission.py:307] 29) loss = 0.647, grad_norm = 0.500 +I0915 07:21:54.264918 140277943293696 logging_writer.py:48] [30] global_step=30, grad_norm=0.5, loss=0.640619 +I0915 07:21:54.267771 140312702207168 submission.py:307] 30) loss = 0.641, grad_norm = 0.500 +I0915 07:21:54.512772 140277951686400 logging_writer.py:48] [31] global_step=31, grad_norm=0.5, loss=0.628566 +I0915 07:21:54.515609 140312702207168 submission.py:307] 31) loss = 0.629, grad_norm = 0.500 +I0915 07:21:54.759844 140277943293696 logging_writer.py:48] [32] global_step=32, grad_norm=0.5, loss=0.620792 +I0915 07:21:54.762674 140312702207168 submission.py:307] 32) loss = 0.621, grad_norm = 0.500 +I0915 07:21:55.006999 140277951686400 logging_writer.py:48] [33] global_step=33, grad_norm=0.5, loss=0.612059 +I0915 07:21:55.009838 140312702207168 submission.py:307] 33) loss = 0.612, grad_norm = 0.500 +I0915 07:21:55.253599 140277943293696 logging_writer.py:48] [34] global_step=34, grad_norm=0.5, loss=0.601812 +I0915 07:21:55.256378 140312702207168 submission.py:307] 34) loss = 0.602, grad_norm = 0.500 +I0915 07:21:55.501018 140277951686400 logging_writer.py:48] [35] global_step=35, grad_norm=0.5, loss=0.591924 +I0915 07:21:55.503843 140312702207168 submission.py:307] 35) loss = 0.592, grad_norm = 0.500 +I0915 07:21:55.749194 140277943293696 logging_writer.py:48] [36] global_step=36, grad_norm=0.5, loss=0.583994 +I0915 07:21:55.752120 140312702207168 submission.py:307] 36) loss = 0.584, grad_norm = 0.500 +I0915 07:21:55.998320 140277951686400 logging_writer.py:48] [37] global_step=37, grad_norm=0.5, loss=0.57771 +I0915 07:21:56.001722 140312702207168 submission.py:307] 37) loss = 0.578, grad_norm = 0.500 +I0915 07:21:56.247244 140277943293696 logging_writer.py:48] [38] global_step=38, grad_norm=0.499999, loss=0.570185 +I0915 07:21:56.250114 140312702207168 submission.py:307] 38) loss = 0.570, grad_norm = 0.500 +I0915 07:21:56.493799 140277951686400 logging_writer.py:48] [39] global_step=39, grad_norm=0.499999, loss=0.56113 +I0915 07:21:56.496672 140312702207168 submission.py:307] 39) loss = 0.561, grad_norm = 0.500 +I0915 07:21:56.742450 140277943293696 logging_writer.py:48] [40] global_step=40, grad_norm=0.499999, loss=0.551384 +I0915 07:21:56.746091 140312702207168 submission.py:307] 40) loss = 0.551, grad_norm = 0.500 +I0915 07:21:56.991933 140277951686400 logging_writer.py:48] [41] global_step=41, grad_norm=0.499999, loss=0.544543 +I0915 07:21:56.994790 140312702207168 submission.py:307] 41) loss = 0.545, grad_norm = 0.500 +I0915 07:21:57.243777 140277943293696 logging_writer.py:48] [42] global_step=42, grad_norm=0.499999, loss=0.53798 +I0915 07:21:57.247030 140312702207168 submission.py:307] 42) loss = 0.538, grad_norm = 0.500 +I0915 07:21:57.494194 140277951686400 logging_writer.py:48] [43] global_step=43, grad_norm=0.499999, loss=0.532264 +I0915 07:21:57.497259 140312702207168 submission.py:307] 43) loss = 0.532, grad_norm = 0.500 +I0915 07:21:57.742824 140277943293696 logging_writer.py:48] [44] global_step=44, grad_norm=0.499999, loss=0.525194 +I0915 07:21:57.745730 140312702207168 submission.py:307] 44) loss = 0.525, grad_norm = 0.500 +I0915 07:21:57.991776 140277951686400 logging_writer.py:48] [45] global_step=45, grad_norm=0.499999, loss=0.518136 +I0915 07:21:57.995091 140312702207168 submission.py:307] 45) loss = 0.518, grad_norm = 0.500 +I0915 07:21:58.238560 140277943293696 logging_writer.py:48] [46] global_step=46, grad_norm=0.499999, loss=0.510621 +I0915 07:21:58.241526 140312702207168 submission.py:307] 46) loss = 0.511, grad_norm = 0.500 +I0915 07:21:58.486653 140277951686400 logging_writer.py:48] [47] global_step=47, grad_norm=0.499999, loss=0.504168 +I0915 07:21:58.489580 140312702207168 submission.py:307] 47) loss = 0.504, grad_norm = 0.500 +I0915 07:21:58.734743 140277943293696 logging_writer.py:48] [48] global_step=48, grad_norm=0.499999, loss=0.500786 +I0915 07:21:58.737725 140312702207168 submission.py:307] 48) loss = 0.501, grad_norm = 0.500 +I0915 07:21:58.983758 140277951686400 logging_writer.py:48] [49] global_step=49, grad_norm=0.499999, loss=0.494561 +I0915 07:21:58.986575 140312702207168 submission.py:307] 49) loss = 0.495, grad_norm = 0.500 +I0915 07:21:59.232160 140277943293696 logging_writer.py:48] [50] global_step=50, grad_norm=0.499999, loss=0.486779 +I0915 07:21:59.235112 140312702207168 submission.py:307] 50) loss = 0.487, grad_norm = 0.500 +I0915 07:21:59.481677 140277951686400 logging_writer.py:48] [51] global_step=51, grad_norm=0.499999, loss=0.48022 +I0915 07:21:59.484514 140312702207168 submission.py:307] 51) loss = 0.480, grad_norm = 0.500 +I0915 07:21:59.730894 140277943293696 logging_writer.py:48] [52] global_step=52, grad_norm=0.499999, loss=0.475135 +I0915 07:21:59.733830 140312702207168 submission.py:307] 52) loss = 0.475, grad_norm = 0.500 +I0915 07:21:59.978356 140277951686400 logging_writer.py:48] [53] global_step=53, grad_norm=0.499999, loss=0.470266 +I0915 07:21:59.981163 140312702207168 submission.py:307] 53) loss = 0.470, grad_norm = 0.500 +I0915 07:22:00.224054 140277943293696 logging_writer.py:48] [54] global_step=54, grad_norm=0.499999, loss=0.465687 +I0915 07:22:00.226950 140312702207168 submission.py:307] 54) loss = 0.466, grad_norm = 0.500 +I0915 07:22:00.472714 140277951686400 logging_writer.py:48] [55] global_step=55, grad_norm=0.499999, loss=0.45994 +I0915 07:22:00.475582 140312702207168 submission.py:307] 55) loss = 0.460, grad_norm = 0.500 +I0915 07:22:00.721115 140277943293696 logging_writer.py:48] [56] global_step=56, grad_norm=0.499999, loss=0.453491 +I0915 07:22:00.724023 140312702207168 submission.py:307] 56) loss = 0.453, grad_norm = 0.500 +I0915 07:22:00.970198 140277951686400 logging_writer.py:48] [57] global_step=57, grad_norm=0.499999, loss=0.449453 +I0915 07:22:00.973042 140312702207168 submission.py:307] 57) loss = 0.449, grad_norm = 0.500 +I0915 07:22:01.215837 140277943293696 logging_writer.py:48] [58] global_step=58, grad_norm=0.499999, loss=0.446646 +I0915 07:22:01.218844 140312702207168 submission.py:307] 58) loss = 0.447, grad_norm = 0.500 +I0915 07:22:01.461063 140277951686400 logging_writer.py:48] [59] global_step=59, grad_norm=0.499999, loss=0.440182 +I0915 07:22:01.463935 140312702207168 submission.py:307] 59) loss = 0.440, grad_norm = 0.500 +I0915 07:22:01.704831 140277943293696 logging_writer.py:48] [60] global_step=60, grad_norm=0.499999, loss=0.434574 +I0915 07:22:01.707683 140312702207168 submission.py:307] 60) loss = 0.435, grad_norm = 0.500 +I0915 07:22:01.951884 140277951686400 logging_writer.py:48] [61] global_step=61, grad_norm=0.499999, loss=0.431182 +I0915 07:22:01.954821 140312702207168 submission.py:307] 61) loss = 0.431, grad_norm = 0.500 +I0915 07:22:02.198059 140277943293696 logging_writer.py:48] [62] global_step=62, grad_norm=0.496148, loss=0.429091 +I0915 07:22:02.200844 140312702207168 submission.py:307] 62) loss = 0.429, grad_norm = 0.496 +I0915 07:22:02.441946 140277951686400 logging_writer.py:48] [63] global_step=63, grad_norm=0.483625, loss=0.426461 +I0915 07:22:02.444754 140312702207168 submission.py:307] 63) loss = 0.426, grad_norm = 0.484 +I0915 07:22:02.687092 140277943293696 logging_writer.py:48] [64] global_step=64, grad_norm=0.479497, loss=0.420042 +I0915 07:22:02.689979 140312702207168 submission.py:307] 64) loss = 0.420, grad_norm = 0.479 +I0915 07:22:02.934065 140277951686400 logging_writer.py:48] [65] global_step=65, grad_norm=0.472093, loss=0.414567 +I0915 07:22:02.936895 140312702207168 submission.py:307] 65) loss = 0.415, grad_norm = 0.472 +I0915 07:22:03.182894 140277943293696 logging_writer.py:48] [66] global_step=66, grad_norm=0.472103, loss=0.412548 +I0915 07:22:03.185748 140312702207168 submission.py:307] 66) loss = 0.413, grad_norm = 0.472 +I0915 07:22:03.432810 140277951686400 logging_writer.py:48] [67] global_step=67, grad_norm=0.463516, loss=0.412142 +I0915 07:22:03.435665 140312702207168 submission.py:307] 67) loss = 0.412, grad_norm = 0.464 +I0915 07:22:03.681401 140277943293696 logging_writer.py:48] [68] global_step=68, grad_norm=0.461954, loss=0.407236 +I0915 07:22:03.684259 140312702207168 submission.py:307] 68) loss = 0.407, grad_norm = 0.462 +I0915 07:22:03.930345 140277951686400 logging_writer.py:48] [69] global_step=69, grad_norm=0.455518, loss=0.402544 +I0915 07:22:03.933163 140312702207168 submission.py:307] 69) loss = 0.403, grad_norm = 0.456 +I0915 07:22:04.179016 140277943293696 logging_writer.py:48] [70] global_step=70, grad_norm=0.452919, loss=0.397313 +I0915 07:22:04.181937 140312702207168 submission.py:307] 70) loss = 0.397, grad_norm = 0.453 +I0915 07:22:04.429155 140277951686400 logging_writer.py:48] [71] global_step=71, grad_norm=0.456101, loss=0.399292 +I0915 07:22:04.432035 140312702207168 submission.py:307] 71) loss = 0.399, grad_norm = 0.456 +I0915 07:22:04.678832 140277943293696 logging_writer.py:48] [72] global_step=72, grad_norm=0.456792, loss=0.396285 +I0915 07:22:04.681766 140312702207168 submission.py:307] 72) loss = 0.396, grad_norm = 0.457 +I0915 07:22:04.924959 140277951686400 logging_writer.py:48] [73] global_step=73, grad_norm=0.454279, loss=0.390801 +I0915 07:22:04.927765 140312702207168 submission.py:307] 73) loss = 0.391, grad_norm = 0.454 +I0915 07:22:05.173927 140277943293696 logging_writer.py:48] [74] global_step=74, grad_norm=0.446615, loss=0.388856 +I0915 07:22:05.176774 140312702207168 submission.py:307] 74) loss = 0.389, grad_norm = 0.447 +I0915 07:22:05.423874 140277951686400 logging_writer.py:48] [75] global_step=75, grad_norm=0.438866, loss=0.387507 +I0915 07:22:05.426841 140312702207168 submission.py:307] 75) loss = 0.388, grad_norm = 0.439 +I0915 07:22:05.672310 140277943293696 logging_writer.py:48] [76] global_step=76, grad_norm=0.434526, loss=0.384443 +I0915 07:22:05.675170 140312702207168 submission.py:307] 76) loss = 0.384, grad_norm = 0.435 +I0915 07:22:05.917672 140277951686400 logging_writer.py:48] [77] global_step=77, grad_norm=0.427372, loss=0.381069 +I0915 07:22:05.920484 140312702207168 submission.py:307] 77) loss = 0.381, grad_norm = 0.427 +I0915 07:22:06.165416 140277943293696 logging_writer.py:48] [78] global_step=78, grad_norm=0.419601, loss=0.380269 +I0915 07:22:06.168231 140312702207168 submission.py:307] 78) loss = 0.380, grad_norm = 0.420 +I0915 07:22:06.416116 140277951686400 logging_writer.py:48] [79] global_step=79, grad_norm=0.412122, loss=0.377145 +I0915 07:22:06.418974 140312702207168 submission.py:307] 79) loss = 0.377, grad_norm = 0.412 +I0915 07:22:06.666495 140277943293696 logging_writer.py:48] [80] global_step=80, grad_norm=0.409314, loss=0.3727 +I0915 07:22:06.669350 140312702207168 submission.py:307] 80) loss = 0.373, grad_norm = 0.409 +I0915 07:22:06.915106 140277951686400 logging_writer.py:48] [81] global_step=81, grad_norm=0.405152, loss=0.37364 +I0915 07:22:06.917931 140312702207168 submission.py:307] 81) loss = 0.374, grad_norm = 0.405 +I0915 07:22:07.163533 140277943293696 logging_writer.py:48] [82] global_step=82, grad_norm=0.40119, loss=0.369678 +I0915 07:22:07.166362 140312702207168 submission.py:307] 82) loss = 0.370, grad_norm = 0.401 +I0915 07:22:07.410759 140277951686400 logging_writer.py:48] [83] global_step=83, grad_norm=0.399934, loss=0.366194 +I0915 07:22:07.413634 140312702207168 submission.py:307] 83) loss = 0.366, grad_norm = 0.400 +I0915 07:22:07.658713 140277943293696 logging_writer.py:48] [84] global_step=84, grad_norm=0.398815, loss=0.366394 +I0915 07:22:07.661553 140312702207168 submission.py:307] 84) loss = 0.366, grad_norm = 0.399 +I0915 07:22:07.908260 140277951686400 logging_writer.py:48] [85] global_step=85, grad_norm=0.39415, loss=0.363832 +I0915 07:22:07.911103 140312702207168 submission.py:307] 85) loss = 0.364, grad_norm = 0.394 +I0915 07:22:08.155864 140277943293696 logging_writer.py:48] [86] global_step=86, grad_norm=0.391751, loss=0.363634 +I0915 07:22:08.158765 140312702207168 submission.py:307] 86) loss = 0.364, grad_norm = 0.392 +I0915 07:22:08.405218 140277951686400 logging_writer.py:48] [87] global_step=87, grad_norm=0.388654, loss=0.363208 +I0915 07:22:08.408033 140312702207168 submission.py:307] 87) loss = 0.363, grad_norm = 0.389 +I0915 07:22:08.658026 140277943293696 logging_writer.py:48] [88] global_step=88, grad_norm=0.388415, loss=0.358074 +I0915 07:22:08.661082 140312702207168 submission.py:307] 88) loss = 0.358, grad_norm = 0.388 +I0915 07:22:08.900249 140277951686400 logging_writer.py:48] [89] global_step=89, grad_norm=0.389688, loss=0.354866 +I0915 07:22:08.903101 140312702207168 submission.py:307] 89) loss = 0.355, grad_norm = 0.390 +I0915 07:22:09.143560 140277943293696 logging_writer.py:48] [90] global_step=90, grad_norm=0.383353, loss=0.353753 +I0915 07:22:09.146471 140312702207168 submission.py:307] 90) loss = 0.354, grad_norm = 0.383 +I0915 07:22:09.390974 140277951686400 logging_writer.py:48] [91] global_step=91, grad_norm=0.380016, loss=0.354722 +I0915 07:22:09.393844 140312702207168 submission.py:307] 91) loss = 0.355, grad_norm = 0.380 +I0915 07:22:09.636790 140277943293696 logging_writer.py:48] [92] global_step=92, grad_norm=0.382302, loss=0.350798 +I0915 07:22:09.639726 140312702207168 submission.py:307] 92) loss = 0.351, grad_norm = 0.382 +I0915 07:22:09.886827 140277951686400 logging_writer.py:48] [93] global_step=93, grad_norm=0.380965, loss=0.350499 +I0915 07:22:09.889742 140312702207168 submission.py:307] 93) loss = 0.350, grad_norm = 0.381 +I0915 07:22:10.132371 140277943293696 logging_writer.py:48] [94] global_step=94, grad_norm=0.375122, loss=0.349146 +I0915 07:22:10.135337 140312702207168 submission.py:307] 94) loss = 0.349, grad_norm = 0.375 +I0915 07:22:10.381040 140277951686400 logging_writer.py:48] [95] global_step=95, grad_norm=0.378459, loss=0.344816 +I0915 07:22:10.383889 140312702207168 submission.py:307] 95) loss = 0.345, grad_norm = 0.378 +I0915 07:22:10.630648 140277943293696 logging_writer.py:48] [96] global_step=96, grad_norm=0.374854, loss=0.344714 +I0915 07:22:10.633527 140312702207168 submission.py:307] 96) loss = 0.345, grad_norm = 0.375 +I0915 07:22:10.877471 140277951686400 logging_writer.py:48] [97] global_step=97, grad_norm=0.37343, loss=0.343883 +I0915 07:22:10.880254 140312702207168 submission.py:307] 97) loss = 0.344, grad_norm = 0.373 +I0915 07:22:11.123954 140277943293696 logging_writer.py:48] [98] global_step=98, grad_norm=0.37347, loss=0.340355 +I0915 07:22:11.126777 140312702207168 submission.py:307] 98) loss = 0.340, grad_norm = 0.373 +I0915 07:22:11.372523 140277951686400 logging_writer.py:48] [99] global_step=99, grad_norm=0.36913, loss=0.342889 +I0915 07:22:11.375375 140312702207168 submission.py:307] 99) loss = 0.343, grad_norm = 0.369 +I0915 07:22:11.617801 140277943293696 logging_writer.py:48] [100] global_step=100, grad_norm=0.37287, loss=0.338563 +I0915 07:22:11.620662 140312702207168 submission.py:307] 100) loss = 0.339, grad_norm = 0.373 +I0915 07:23:45.238884 140277951686400 logging_writer.py:48] [500] global_step=500, grad_norm=0.0325641, loss=0.0646931 +I0915 07:23:45.242152 140312702207168 submission.py:307] 500) loss = 0.065, grad_norm = 0.033 +I0915 07:25:41.395191 140277943293696 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.0143857, loss=0.0496285 +I0915 07:25:41.398471 140312702207168 submission.py:307] 1000) loss = 0.050, grad_norm = 0.014 +I0915 07:27:37.121929 140277951686400 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.0598499, loss=0.0498266 +I0915 07:27:37.125332 140312702207168 submission.py:307] 1500) loss = 0.050, grad_norm = 0.060 +I0915 07:29:19.000343 140312702207168 spec.py:333] Evaluating on the training split. +I0915 07:29:45.104977 140312702207168 spec.py:346] Evaluating on the validation split. +I0915 07:29:47.094707 140312702207168 spec.py:363] Evaluating on the test split. +I0915 07:29:49.127379 140312702207168 submission_runner.py:516] Time since start: 602.80s, Step: 1936, {'train/accuracy': 0.9874387078307416, 'train/loss': 0.046041251289779046, 'train/mean_average_precision': 0.10991815016090035, 'validation/accuracy': 0.9846562549725757, 'validation/loss': 0.05524435237173596, 'validation/mean_average_precision': 0.11032133945178922, 'validation/num_examples': 43793, 'test/accuracy': 0.983686098586721, 'test/loss': 0.05844512193837972, 'test/mean_average_precision': 0.10675829297918207, 'test/num_examples': 43793, 'score': 490.7094373703003, 'total_duration': 602.7956111431122, 'accumulated_submission_time': 490.7094373703003, 'accumulated_eval_time': 110.27350735664368, 'accumulated_logging_time': 0.030091047286987305} +I0915 07:29:49.152209 140277943293696 logging_writer.py:48] [1936] accumulated_eval_time=110.274, accumulated_logging_time=0.030091, accumulated_submission_time=490.709, global_step=1936, preemption_count=0, score=490.709, test/accuracy=0.983686, test/loss=0.0584451, test/mean_average_precision=0.106758, test/num_examples=43793, total_duration=602.796, train/accuracy=0.987439, train/loss=0.0460413, train/mean_average_precision=0.109918, validation/accuracy=0.984656, validation/loss=0.0552444, validation/mean_average_precision=0.110321, validation/num_examples=43793 +I0915 07:30:04.827142 140277951686400 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.0281626, loss=0.0442164 +I0915 07:30:04.830179 140312702207168 submission.py:307] 2000) loss = 0.044, grad_norm = 0.028 +I0915 07:32:01.832419 140277943293696 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.0333439, loss=0.0416483 +I0915 07:32:01.835864 140312702207168 submission.py:307] 2500) loss = 0.042, grad_norm = 0.033 +I0915 07:33:59.466020 140277951686400 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.0156122, loss=0.0440174 +I0915 07:33:59.469787 140312702207168 submission.py:307] 3000) loss = 0.044, grad_norm = 0.016 +I0915 07:35:56.406668 140277943293696 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.0248397, loss=0.0414242 +I0915 07:35:56.410002 140312702207168 submission.py:307] 3500) loss = 0.041, grad_norm = 0.025 +I0915 07:37:21.645048 140312702207168 spec.py:333] Evaluating on the training split. +I0915 07:37:48.028112 140312702207168 spec.py:346] Evaluating on the validation split. +I0915 07:37:50.060362 140312702207168 spec.py:363] Evaluating on the test split. +I0915 07:37:52.081968 140312702207168 submission_runner.py:516] Time since start: 1085.75s, Step: 3864, {'train/accuracy': 0.9882669022075653, 'train/loss': 0.04080821941564888, 'train/mean_average_precision': 0.18901202435626965, 'validation/accuracy': 0.9853883398984864, 'validation/loss': 0.05039437853357321, 'validation/mean_average_precision': 0.16237232614094704, 'validation/num_examples': 43793, 'test/accuracy': 0.9845300735572198, 'test/loss': 0.05313986708341791, 'test/mean_average_precision': 0.16781257602087565, 'test/num_examples': 43793, 'score': 941.8389570713043, 'total_duration': 1085.7501618862152, 'accumulated_submission_time': 941.8389570713043, 'accumulated_eval_time': 140.7103328704834, 'accumulated_logging_time': 0.06471657752990723} +I0915 07:37:52.106921 140277951686400 logging_writer.py:48] [3864] accumulated_eval_time=140.71, accumulated_logging_time=0.0647166, accumulated_submission_time=941.839, global_step=3864, preemption_count=0, score=941.839, test/accuracy=0.98453, test/loss=0.0531399, test/mean_average_precision=0.167813, test/num_examples=43793, total_duration=1085.75, train/accuracy=0.988267, train/loss=0.0408082, train/mean_average_precision=0.189012, validation/accuracy=0.985388, validation/loss=0.0503944, validation/mean_average_precision=0.162372, validation/num_examples=43793 +I0915 07:38:24.519899 140277943293696 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.0239528, loss=0.0421222 +I0915 07:38:24.523064 140312702207168 submission.py:307] 4000) loss = 0.042, grad_norm = 0.024 +I0915 07:40:21.886600 140277951686400 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.0258109, loss=0.0439054 +I0915 07:40:21.889970 140312702207168 submission.py:307] 4500) loss = 0.044, grad_norm = 0.026 +I0915 07:42:19.084382 140277943293696 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.0187117, loss=0.0388878 +I0915 07:42:19.087748 140312702207168 submission.py:307] 5000) loss = 0.039, grad_norm = 0.019 +I0915 07:44:15.817725 140277951686400 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.0210042, loss=0.0417107 +I0915 07:44:15.820931 140312702207168 submission.py:307] 5500) loss = 0.042, grad_norm = 0.021 +I0915 07:45:24.606806 140312702207168 spec.py:333] Evaluating on the training split. +I0915 07:45:50.795383 140312702207168 spec.py:346] Evaluating on the validation split. +I0915 07:45:52.788690 140312702207168 spec.py:363] Evaluating on the test split. +I0915 07:45:54.869020 140312702207168 submission_runner.py:516] Time since start: 1568.54s, Step: 5794, {'train/accuracy': 0.989055488586244, 'train/loss': 0.03738072198791166, 'train/mean_average_precision': 0.2414374545903787, 'validation/accuracy': 0.9858494701872773, 'validation/loss': 0.047440085029015484, 'validation/mean_average_precision': 0.19876964275093487, 'validation/num_examples': 43793, 'test/accuracy': 0.9850093367889701, 'test/loss': 0.05008736393851617, 'test/mean_average_precision': 0.20115020681102608, 'test/num_examples': 43793, 'score': 1392.9659972190857, 'total_duration': 1568.5372149944305, 'accumulated_submission_time': 1392.9659972190857, 'accumulated_eval_time': 170.97252202033997, 'accumulated_logging_time': 0.09939122200012207} +I0915 07:45:54.894287 140277943293696 logging_writer.py:48] [5794] accumulated_eval_time=170.973, accumulated_logging_time=0.0993912, accumulated_submission_time=1392.97, global_step=5794, preemption_count=0, score=1392.97, test/accuracy=0.985009, test/loss=0.0500874, test/mean_average_precision=0.20115, test/num_examples=43793, total_duration=1568.54, train/accuracy=0.989055, train/loss=0.0373807, train/mean_average_precision=0.241437, validation/accuracy=0.985849, validation/loss=0.0474401, validation/mean_average_precision=0.19877, validation/num_examples=43793 +I0915 07:46:43.870328 140277951686400 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.0230163, loss=0.0389812 +I0915 07:46:43.873406 140312702207168 submission.py:307] 6000) loss = 0.039, grad_norm = 0.023 +I0915 07:48:40.601848 140277943293696 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.0313803, loss=0.0370673 +I0915 07:48:40.605075 140312702207168 submission.py:307] 6500) loss = 0.037, grad_norm = 0.031 +I0915 07:50:37.754621 140277951686400 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.02087, loss=0.0417549 +I0915 07:50:37.757972 140312702207168 submission.py:307] 7000) loss = 0.042, grad_norm = 0.021 +I0915 07:52:33.405277 140277943293696 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.0230512, loss=0.0393678 +I0915 07:52:33.411986 140312702207168 submission.py:307] 7500) loss = 0.039, grad_norm = 0.023 +I0915 07:53:27.519096 140312702207168 spec.py:333] Evaluating on the training split. +I0915 07:53:53.164480 140312702207168 spec.py:346] Evaluating on the validation split. +I0915 07:53:55.158112 140312702207168 spec.py:363] Evaluating on the test split. +I0915 07:53:57.190021 140312702207168 submission_runner.py:516] Time since start: 2050.86s, Step: 7737, {'train/accuracy': 0.9896002426085613, 'train/loss': 0.035199966950090214, 'train/mean_average_precision': 0.2977852960696569, 'validation/accuracy': 0.9863487573837676, 'validation/loss': 0.04575225709271966, 'validation/mean_average_precision': 0.23230311817155064, 'validation/num_examples': 43793, 'test/accuracy': 0.9855458757672213, 'test/loss': 0.04839842364827133, 'test/mean_average_precision': 0.2330750212756477, 'test/num_examples': 43793, 'score': 1844.2267653942108, 'total_duration': 2050.8582327365875, 'accumulated_submission_time': 1844.2267653942108, 'accumulated_eval_time': 200.64341568946838, 'accumulated_logging_time': 0.1341695785522461} +I0915 07:53:57.213748 140277951686400 logging_writer.py:48] [7737] accumulated_eval_time=200.643, accumulated_logging_time=0.13417, accumulated_submission_time=1844.23, global_step=7737, preemption_count=0, score=1844.23, test/accuracy=0.985546, test/loss=0.0483984, test/mean_average_precision=0.233075, test/num_examples=43793, total_duration=2050.86, train/accuracy=0.9896, train/loss=0.0352, train/mean_average_precision=0.297785, validation/accuracy=0.986349, validation/loss=0.0457523, validation/mean_average_precision=0.232303, validation/num_examples=43793 +I0915 07:54:58.163341 140277943293696 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.0199957, loss=0.0353456 +I0915 07:54:58.166273 140312702207168 submission.py:307] 8000) loss = 0.035, grad_norm = 0.020 +I0915 07:56:50.355641 140277951686400 logging_writer.py:48] [8500] global_step=8500, grad_norm=0.016203, loss=0.0346913 +I0915 07:56:50.358907 140312702207168 submission.py:307] 8500) loss = 0.035, grad_norm = 0.016 +I0915 07:58:42.671471 140277943293696 logging_writer.py:48] [9000] global_step=9000, grad_norm=0.014801, loss=0.0346018 +I0915 07:58:42.674787 140312702207168 submission.py:307] 9000) loss = 0.035, grad_norm = 0.015 +I0915 08:00:33.546924 140277951686400 logging_writer.py:48] [9500] global_step=9500, grad_norm=0.0206445, loss=0.0329052 +I0915 08:00:33.550167 140312702207168 submission.py:307] 9500) loss = 0.033, grad_norm = 0.021 +I0915 08:01:29.762273 140312702207168 spec.py:333] Evaluating on the training split. +I0915 08:01:54.634089 140312702207168 spec.py:346] Evaluating on the validation split. +I0915 08:01:56.627102 140312702207168 spec.py:363] Evaluating on the test split. +I0915 08:01:58.675853 140312702207168 submission_runner.py:516] Time since start: 2532.34s, Step: 9754, {'train/accuracy': 0.9903221802173521, 'train/loss': 0.03253789438418908, 'train/mean_average_precision': 0.35303903570077905, 'validation/accuracy': 0.9864611984841154, 'validation/loss': 0.045217572006955926, 'validation/mean_average_precision': 0.24381467199369627, 'validation/num_examples': 43793, 'test/accuracy': 0.9856402122908698, 'test/loss': 0.047726486489872724, 'test/mean_average_precision': 0.2528231426107782, 'test/num_examples': 43793, 'score': 2295.3961102962494, 'total_duration': 2532.3440577983856, 'accumulated_submission_time': 2295.3961102962494, 'accumulated_eval_time': 229.55694007873535, 'accumulated_logging_time': 0.16705751419067383} +I0915 08:01:58.698761 140277943293696 logging_writer.py:48] [9754] accumulated_eval_time=229.557, accumulated_logging_time=0.167058, accumulated_submission_time=2295.4, global_step=9754, preemption_count=0, score=2295.4, test/accuracy=0.98564, test/loss=0.0477265, test/mean_average_precision=0.252823, test/num_examples=43793, total_duration=2532.34, train/accuracy=0.990322, train/loss=0.0325379, train/mean_average_precision=0.353039, validation/accuracy=0.986461, validation/loss=0.0452176, validation/mean_average_precision=0.243815, validation/num_examples=43793 +I0915 08:02:53.992845 140277951686400 logging_writer.py:48] [10000] global_step=10000, grad_norm=0.0196784, loss=0.0348735 +I0915 08:02:53.995984 140312702207168 submission.py:307] 10000) loss = 0.035, grad_norm = 0.020 +I0915 08:04:44.779191 140277943293696 logging_writer.py:48] [10500] global_step=10500, grad_norm=0.0182831, loss=0.034425 +I0915 08:04:44.782448 140312702207168 submission.py:307] 10500) loss = 0.034, grad_norm = 0.018 +I0915 08:06:35.389529 140277951686400 logging_writer.py:48] [11000] global_step=11000, grad_norm=0.0165124, loss=0.0324084 +I0915 08:06:35.392707 140312702207168 submission.py:307] 11000) loss = 0.032, grad_norm = 0.017 +I0915 08:08:25.981701 140277943293696 logging_writer.py:48] [11500] global_step=11500, grad_norm=0.017331, loss=0.0325642 +I0915 08:08:25.984916 140312702207168 submission.py:307] 11500) loss = 0.033, grad_norm = 0.017 +I0915 08:09:31.101360 140312702207168 spec.py:333] Evaluating on the training split. +I0915 08:09:56.042322 140312702207168 spec.py:346] Evaluating on the validation split. +I0915 08:09:58.001101 140312702207168 spec.py:363] Evaluating on the test split. +I0915 08:10:00.026713 140312702207168 submission_runner.py:516] Time since start: 3013.69s, Step: 11792, {'train/accuracy': 0.9906539829082384, 'train/loss': 0.031130412809663867, 'train/mean_average_precision': 0.3894310935237193, 'validation/accuracy': 0.9867177427997104, 'validation/loss': 0.044264898800980064, 'validation/mean_average_precision': 0.26252952560447534, 'validation/num_examples': 43793, 'test/accuracy': 0.9859383830888304, 'test/loss': 0.047000152507203685, 'test/mean_average_precision': 0.2628282831636894, 'test/num_examples': 43793, 'score': 2746.4137926101685, 'total_duration': 3013.6949014663696, 'accumulated_submission_time': 2746.4137926101685, 'accumulated_eval_time': 258.4821677207947, 'accumulated_logging_time': 0.19896602630615234} +I0915 08:10:00.050765 140277951686400 logging_writer.py:48] [11792] accumulated_eval_time=258.482, accumulated_logging_time=0.198966, accumulated_submission_time=2746.41, global_step=11792, preemption_count=0, score=2746.41, test/accuracy=0.985938, test/loss=0.0470002, test/mean_average_precision=0.262828, test/num_examples=43793, total_duration=3013.69, train/accuracy=0.990654, train/loss=0.0311304, train/mean_average_precision=0.389431, validation/accuracy=0.986718, validation/loss=0.0442649, validation/mean_average_precision=0.26253, validation/num_examples=43793 +I0915 08:10:46.968318 140277943293696 logging_writer.py:48] [12000] global_step=12000, grad_norm=0.0179334, loss=0.0359321 +I0915 08:10:46.971226 140312702207168 submission.py:307] 12000) loss = 0.036, grad_norm = 0.018 +I0915 08:12:37.701017 140277951686400 logging_writer.py:48] [12500] global_step=12500, grad_norm=0.01793, loss=0.034262 +I0915 08:12:37.704107 140312702207168 submission.py:307] 12500) loss = 0.034, grad_norm = 0.018 +I0915 08:14:29.400343 140277943293696 logging_writer.py:48] [13000] global_step=13000, grad_norm=0.0203433, loss=0.0335464 +I0915 08:14:29.406049 140312702207168 submission.py:307] 13000) loss = 0.034, grad_norm = 0.020 +I0915 08:16:19.958444 140277951686400 logging_writer.py:48] [13500] global_step=13500, grad_norm=0.0204059, loss=0.0337169 +I0915 08:16:19.961743 140312702207168 submission.py:307] 13500) loss = 0.034, grad_norm = 0.020 +I0915 08:17:32.631020 140312702207168 spec.py:333] Evaluating on the training split. +I0915 08:17:57.338813 140312702207168 spec.py:346] Evaluating on the validation split. +I0915 08:17:59.347807 140312702207168 spec.py:363] Evaluating on the test split. +I0915 08:18:01.361110 140312702207168 submission_runner.py:516] Time since start: 3495.03s, Step: 13827, {'train/accuracy': 0.990904449564144, 'train/loss': 0.030062703453749534, 'train/mean_average_precision': 0.40850109324061173, 'validation/accuracy': 0.9865992128311126, 'validation/loss': 0.04483021216255492, 'validation/mean_average_precision': 0.25851990515393536, 'validation/num_examples': 43793, 'test/accuracy': 0.9858861610846679, 'test/loss': 0.04735655978914102, 'test/mean_average_precision': 0.25399398129500506, 'test/num_examples': 43793, 'score': 3197.621773481369, 'total_duration': 3495.029235601425, 'accumulated_submission_time': 3197.621773481369, 'accumulated_eval_time': 287.21206617355347, 'accumulated_logging_time': 0.23195123672485352} +I0915 08:18:01.388419 140277943293696 logging_writer.py:48] [13827] accumulated_eval_time=287.212, accumulated_logging_time=0.231951, accumulated_submission_time=3197.62, global_step=13827, preemption_count=0, score=3197.62, test/accuracy=0.985886, test/loss=0.0473566, test/mean_average_precision=0.253994, test/num_examples=43793, total_duration=3495.03, train/accuracy=0.990904, train/loss=0.0300627, train/mean_average_precision=0.408501, validation/accuracy=0.986599, validation/loss=0.0448302, validation/mean_average_precision=0.25852, validation/num_examples=43793 +I0915 08:18:40.293416 140277951686400 logging_writer.py:48] [14000] global_step=14000, grad_norm=0.0169493, loss=0.0330674 +I0915 08:18:40.296670 140312702207168 submission.py:307] 14000) loss = 0.033, grad_norm = 0.017 +I0915 08:20:31.274044 140277943293696 logging_writer.py:48] [14500] global_step=14500, grad_norm=0.0170507, loss=0.0312016 +I0915 08:20:31.277078 140312702207168 submission.py:307] 14500) loss = 0.031, grad_norm = 0.017 +I0915 08:22:22.447635 140277951686400 logging_writer.py:48] [15000] global_step=15000, grad_norm=0.0191249, loss=0.0290721 +I0915 08:22:22.450972 140312702207168 submission.py:307] 15000) loss = 0.029, grad_norm = 0.019 +I0915 08:24:13.157549 140277943293696 logging_writer.py:48] [15500] global_step=15500, grad_norm=0.0231501, loss=0.0319442 +I0915 08:24:13.160705 140312702207168 submission.py:307] 15500) loss = 0.032, grad_norm = 0.023 +I0915 08:25:33.776433 140312702207168 spec.py:333] Evaluating on the training split. +I0915 08:25:58.666814 140312702207168 spec.py:346] Evaluating on the validation split. +I0915 08:26:00.640233 140312702207168 spec.py:363] Evaluating on the test split. +I0915 08:26:02.626125 140312702207168 submission_runner.py:516] Time since start: 3976.29s, Step: 15863, {'train/accuracy': 0.9915098901523317, 'train/loss': 0.027868761313151177, 'train/mean_average_precision': 0.4657407301065514, 'validation/accuracy': 0.9868423616365579, 'validation/loss': 0.04452365033131562, 'validation/mean_average_precision': 0.27412144806822464, 'validation/num_examples': 43793, 'test/accuracy': 0.986044511677935, 'test/loss': 0.047118303475542835, 'test/mean_average_precision': 0.26796642982629504, 'test/num_examples': 43793, 'score': 3648.6258997917175, 'total_duration': 3976.294326543808, 'accumulated_submission_time': 3648.6258997917175, 'accumulated_eval_time': 316.0617251396179, 'accumulated_logging_time': 0.27471208572387695} +I0915 08:26:02.650541 140277951686400 logging_writer.py:48] [15863] accumulated_eval_time=316.062, accumulated_logging_time=0.274712, accumulated_submission_time=3648.63, global_step=15863, preemption_count=0, score=3648.63, test/accuracy=0.986045, test/loss=0.0471183, test/mean_average_precision=0.267966, test/num_examples=43793, total_duration=3976.29, train/accuracy=0.99151, train/loss=0.0278688, train/mean_average_precision=0.465741, validation/accuracy=0.986842, validation/loss=0.0445237, validation/mean_average_precision=0.274121, validation/num_examples=43793 +I0915 08:26:33.797557 140277943293696 logging_writer.py:48] [16000] global_step=16000, grad_norm=0.0188569, loss=0.0304677 +I0915 08:26:33.800441 140312702207168 submission.py:307] 16000) loss = 0.030, grad_norm = 0.019 +I0915 08:28:24.742512 140277951686400 logging_writer.py:48] [16500] global_step=16500, grad_norm=0.0170336, loss=0.0293014 +I0915 08:28:24.745606 140312702207168 submission.py:307] 16500) loss = 0.029, grad_norm = 0.017 +I0915 08:30:16.055621 140277943293696 logging_writer.py:48] [17000] global_step=17000, grad_norm=0.017321, loss=0.026329 +I0915 08:30:16.059582 140312702207168 submission.py:307] 17000) loss = 0.026, grad_norm = 0.017 +I0915 08:32:07.144194 140277951686400 logging_writer.py:48] [17500] global_step=17500, grad_norm=0.0180249, loss=0.0282222 +I0915 08:32:07.147307 140312702207168 submission.py:307] 17500) loss = 0.028, grad_norm = 0.018 +I0915 08:33:35.242670 140312702207168 spec.py:333] Evaluating on the training split. +I0915 08:34:00.058146 140312702207168 spec.py:346] Evaluating on the validation split. +I0915 08:34:02.050331 140312702207168 spec.py:363] Evaluating on the test split. +I0915 08:34:04.022743 140312702207168 submission_runner.py:516] Time since start: 4457.69s, Step: 17900, {'train/accuracy': 0.9922223546765736, 'train/loss': 0.025619514450496976, 'train/mean_average_precision': 0.5099060104916786, 'validation/accuracy': 0.9867916210678089, 'validation/loss': 0.04474032524501606, 'validation/mean_average_precision': 0.27667516709397294, 'validation/num_examples': 43793, 'test/accuracy': 0.9860301927413099, 'test/loss': 0.0473273395510508, 'test/mean_average_precision': 0.26931124195509876, 'test/num_examples': 43793, 'score': 4099.839018344879, 'total_duration': 4457.6909737586975, 'accumulated_submission_time': 4099.839018344879, 'accumulated_eval_time': 344.84174847602844, 'accumulated_logging_time': 0.308274507522583} +I0915 08:34:04.046071 140277943293696 logging_writer.py:48] [17900] accumulated_eval_time=344.842, accumulated_logging_time=0.308275, accumulated_submission_time=4099.84, global_step=17900, preemption_count=0, score=4099.84, test/accuracy=0.98603, test/loss=0.0473273, test/mean_average_precision=0.269311, test/num_examples=43793, total_duration=4457.69, train/accuracy=0.992222, train/loss=0.0256195, train/mean_average_precision=0.509906, validation/accuracy=0.986792, validation/loss=0.0447403, validation/mean_average_precision=0.276675, validation/num_examples=43793 +I0915 08:34:26.829164 140277951686400 logging_writer.py:48] [18000] global_step=18000, grad_norm=0.0268105, loss=0.0282559 +I0915 08:34:26.832021 140312702207168 submission.py:307] 18000) loss = 0.028, grad_norm = 0.027 +I0915 08:36:17.019704 140277943293696 logging_writer.py:48] [18500] global_step=18500, grad_norm=0.0196174, loss=0.0271545 +I0915 08:36:17.022821 140312702207168 submission.py:307] 18500) loss = 0.027, grad_norm = 0.020 +I0915 08:38:08.547698 140277951686400 logging_writer.py:48] [19000] global_step=19000, grad_norm=0.0220597, loss=0.0283864 +I0915 08:38:08.550896 140312702207168 submission.py:307] 19000) loss = 0.028, grad_norm = 0.022 +I0915 08:39:58.909986 140277943293696 logging_writer.py:48] [19500] global_step=19500, grad_norm=0.0220709, loss=0.025642 +I0915 08:39:58.913141 140312702207168 submission.py:307] 19500) loss = 0.026, grad_norm = 0.022 +I0915 08:41:36.555740 140312702207168 spec.py:333] Evaluating on the training split. +I0915 08:42:01.464125 140312702207168 spec.py:346] Evaluating on the validation split. +I0915 08:42:03.462053 140312702207168 spec.py:363] Evaluating on the test split. +I0915 08:42:05.482775 140312702207168 submission_runner.py:516] Time since start: 4939.15s, Step: 19944, {'train/accuracy': 0.9924673468409008, 'train/loss': 0.02455019538728184, 'train/mean_average_precision': 0.5369672115837776, 'validation/accuracy': 0.9868094817480085, 'validation/loss': 0.04524322770702964, 'validation/mean_average_precision': 0.27004415497990425, 'validation/num_examples': 43793, 'test/accuracy': 0.9861186332322304, 'test/loss': 0.04769061347378245, 'test/mean_average_precision': 0.26693627059162134, 'test/num_examples': 43793, 'score': 4550.96763586998, 'total_duration': 4939.150961399078, 'accumulated_submission_time': 4550.96763586998, 'accumulated_eval_time': 373.76872181892395, 'accumulated_logging_time': 0.34072184562683105} +I0915 08:42:05.507103 140277951686400 logging_writer.py:48] [19944] accumulated_eval_time=373.769, accumulated_logging_time=0.340722, accumulated_submission_time=4550.97, global_step=19944, preemption_count=0, score=4550.97, test/accuracy=0.986119, test/loss=0.0476906, test/mean_average_precision=0.266936, test/num_examples=43793, total_duration=4939.15, train/accuracy=0.992467, train/loss=0.0245502, train/mean_average_precision=0.536967, validation/accuracy=0.986809, validation/loss=0.0452432, validation/mean_average_precision=0.270044, validation/num_examples=43793 +I0915 08:42:18.645912 140277943293696 logging_writer.py:48] [20000] global_step=20000, grad_norm=0.0151982, loss=0.0267525 +I0915 08:42:18.648757 140312702207168 submission.py:307] 20000) loss = 0.027, grad_norm = 0.015 +I0915 08:44:10.021832 140277951686400 logging_writer.py:48] [20500] global_step=20500, grad_norm=0.019552, loss=0.0274878 +I0915 08:44:10.025049 140312702207168 submission.py:307] 20500) loss = 0.027, grad_norm = 0.020 +I0915 08:46:01.780130 140277943293696 logging_writer.py:48] [21000] global_step=21000, grad_norm=0.0207613, loss=0.028462 +I0915 08:46:01.783164 140312702207168 submission.py:307] 21000) loss = 0.028, grad_norm = 0.021 +I0915 08:47:53.379642 140277951686400 logging_writer.py:48] [21500] global_step=21500, grad_norm=0.0213205, loss=0.0291241 +I0915 08:47:53.382909 140312702207168 submission.py:307] 21500) loss = 0.029, grad_norm = 0.021 +I0915 08:49:38.054693 140312702207168 spec.py:333] Evaluating on the training split. +I0915 08:50:02.762875 140312702207168 spec.py:346] Evaluating on the validation split. +I0915 08:50:04.718233 140312702207168 spec.py:363] Evaluating on the test split. +I0915 08:50:06.743888 140312702207168 submission_runner.py:516] Time since start: 5420.41s, Step: 21971, {'train/accuracy': 0.9925652639004054, 'train/loss': 0.023963529338936987, 'train/mean_average_precision': 0.5399600387924857, 'validation/accuracy': 0.9868918844316569, 'validation/loss': 0.04545238667398413, 'validation/mean_average_precision': 0.2773576826036955, 'validation/num_examples': 43793, 'test/accuracy': 0.9859952376901365, 'test/loss': 0.04824379755782113, 'test/mean_average_precision': 0.2705550973973591, 'test/num_examples': 43793, 'score': 5002.144770145416, 'total_duration': 5420.412091016769, 'accumulated_submission_time': 5002.144770145416, 'accumulated_eval_time': 402.4578549861908, 'accumulated_logging_time': 0.3742103576660156} +I0915 08:50:06.768127 140277943293696 logging_writer.py:48] [21971] accumulated_eval_time=402.458, accumulated_logging_time=0.37421, accumulated_submission_time=5002.14, global_step=21971, preemption_count=0, score=5002.14, test/accuracy=0.985995, test/loss=0.0482438, test/mean_average_precision=0.270555, test/num_examples=43793, total_duration=5420.41, train/accuracy=0.992565, train/loss=0.0239635, train/mean_average_precision=0.53996, validation/accuracy=0.986892, validation/loss=0.0454524, validation/mean_average_precision=0.277358, validation/num_examples=43793 +I0915 08:50:13.856668 140277951686400 logging_writer.py:48] [22000] global_step=22000, grad_norm=0.0171118, loss=0.0256811 +I0915 08:50:13.859677 140312702207168 submission.py:307] 22000) loss = 0.026, grad_norm = 0.017 +I0915 08:52:03.864319 140277943293696 logging_writer.py:48] [22500] global_step=22500, grad_norm=0.0253694, loss=0.0297632 +I0915 08:52:03.867358 140312702207168 submission.py:307] 22500) loss = 0.030, grad_norm = 0.025 +I0915 08:53:53.718806 140277951686400 logging_writer.py:48] [23000] global_step=23000, grad_norm=0.0207724, loss=0.025907 +I0915 08:53:53.722016 140312702207168 submission.py:307] 23000) loss = 0.026, grad_norm = 0.021 +I0915 08:55:43.513305 140277943293696 logging_writer.py:48] [23500] global_step=23500, grad_norm=0.0217918, loss=0.0261836 +I0915 08:55:43.516695 140312702207168 submission.py:307] 23500) loss = 0.026, grad_norm = 0.022 +I0915 08:57:33.028006 140277951686400 logging_writer.py:48] [24000] global_step=24000, grad_norm=0.0246127, loss=0.0266758 +I0915 08:57:33.031247 140312702207168 submission.py:307] 24000) loss = 0.027, grad_norm = 0.025 +I0915 08:57:39.366929 140312702207168 spec.py:333] Evaluating on the training split. +I0915 08:58:04.246872 140312702207168 spec.py:346] Evaluating on the validation split. +I0915 08:58:06.206056 140312702207168 spec.py:363] Evaluating on the test split. +I0915 08:58:08.201263 140312702207168 submission_runner.py:516] Time since start: 5901.87s, Step: 24028, {'train/accuracy': 0.9929287971617616, 'train/loss': 0.022862690287177863, 'train/mean_average_precision': 0.5828592682832358, 'validation/accuracy': 0.9867441278954598, 'validation/loss': 0.04624532222696703, 'validation/mean_average_precision': 0.2789765513309469, 'validation/num_examples': 43793, 'test/accuracy': 0.9860739918415753, 'test/loss': 0.04891026311888339, 'test/mean_average_precision': 0.27158141453834034, 'test/num_examples': 43793, 'score': 5453.354311943054, 'total_duration': 5901.869510173798, 'accumulated_submission_time': 5453.354311943054, 'accumulated_eval_time': 431.29218077659607, 'accumulated_logging_time': 0.4074556827545166} +I0915 08:58:08.223593 140277943293696 logging_writer.py:48] [24028] accumulated_eval_time=431.292, accumulated_logging_time=0.407456, accumulated_submission_time=5453.35, global_step=24028, preemption_count=0, score=5453.35, test/accuracy=0.986074, test/loss=0.0489103, test/mean_average_precision=0.271581, test/num_examples=43793, total_duration=5901.87, train/accuracy=0.992929, train/loss=0.0228627, train/mean_average_precision=0.582859, validation/accuracy=0.986744, validation/loss=0.0462453, validation/mean_average_precision=0.278977, validation/num_examples=43793 +I0915 08:59:52.954550 140277951686400 logging_writer.py:48] [24500] global_step=24500, grad_norm=0.0206635, loss=0.0261751 +I0915 08:59:52.957843 140312702207168 submission.py:307] 24500) loss = 0.026, grad_norm = 0.021 +I0915 09:01:42.209677 140277943293696 logging_writer.py:48] [25000] global_step=25000, grad_norm=0.0220484, loss=0.0246489 +I0915 09:01:42.212793 140312702207168 submission.py:307] 25000) loss = 0.025, grad_norm = 0.022 +I0915 09:03:31.764787 140277951686400 logging_writer.py:48] [25500] global_step=25500, grad_norm=0.0215553, loss=0.0249975 +I0915 09:03:31.767975 140312702207168 submission.py:307] 25500) loss = 0.025, grad_norm = 0.022 +I0915 09:05:20.850482 140277943293696 logging_writer.py:48] [26000] global_step=26000, grad_norm=0.0249625, loss=0.0234744 +I0915 09:05:20.853744 140312702207168 submission.py:307] 26000) loss = 0.023, grad_norm = 0.025 +I0915 09:05:40.718142 140312702207168 spec.py:333] Evaluating on the training split. +I0915 09:06:05.734816 140312702207168 spec.py:346] Evaluating on the validation split. +I0915 09:06:07.723294 140312702207168 spec.py:363] Evaluating on the test split. +I0915 09:06:09.785030 140312702207168 submission_runner.py:516] Time since start: 6383.45s, Step: 26090, {'train/accuracy': 0.9933762598810584, 'train/loss': 0.021087428305450753, 'train/mean_average_precision': 0.616506450120422, 'validation/accuracy': 0.9867473752918597, 'validation/loss': 0.04698270002845531, 'validation/mean_average_precision': 0.2799395559942953, 'validation/num_examples': 43793, 'test/accuracy': 0.985984709060265, 'test/loss': 0.049807279352556647, 'test/mean_average_precision': 0.2677533639503699, 'test/num_examples': 43793, 'score': 5904.460551261902, 'total_duration': 6383.45326423645, 'accumulated_submission_time': 5904.460551261902, 'accumulated_eval_time': 460.3590455055237, 'accumulated_logging_time': 0.4385535717010498} +I0915 09:06:09.809871 140277951686400 logging_writer.py:48] [26090] accumulated_eval_time=460.359, accumulated_logging_time=0.438554, accumulated_submission_time=5904.46, global_step=26090, preemption_count=0, score=5904.46, test/accuracy=0.985985, test/loss=0.0498073, test/mean_average_precision=0.267753, test/num_examples=43793, total_duration=6383.45, train/accuracy=0.993376, train/loss=0.0210874, train/mean_average_precision=0.616506, validation/accuracy=0.986747, validation/loss=0.0469827, validation/mean_average_precision=0.27994, validation/num_examples=43793 +I0915 09:07:40.080984 140277943293696 logging_writer.py:48] [26500] global_step=26500, grad_norm=0.0208718, loss=0.0239087 +I0915 09:07:40.084057 140312702207168 submission.py:307] 26500) loss = 0.024, grad_norm = 0.021 +I0915 09:09:29.267645 140277951686400 logging_writer.py:48] [27000] global_step=27000, grad_norm=0.0216876, loss=0.0233062 +I0915 09:09:29.270886 140312702207168 submission.py:307] 27000) loss = 0.023, grad_norm = 0.022 +I0915 09:11:18.072484 140277943293696 logging_writer.py:48] [27500] global_step=27500, grad_norm=0.0249716, loss=0.0241279 +I0915 09:11:18.075839 140312702207168 submission.py:307] 27500) loss = 0.024, grad_norm = 0.025 +I0915 09:13:06.995496 140277951686400 logging_writer.py:48] [28000] global_step=28000, grad_norm=0.0216584, loss=0.023487 +I0915 09:13:06.998791 140312702207168 submission.py:307] 28000) loss = 0.023, grad_norm = 0.022 +I0915 09:13:42.201676 140312702207168 spec.py:333] Evaluating on the training split. +I0915 09:14:07.026787 140312702207168 spec.py:346] Evaluating on the validation split. +I0915 09:14:08.995158 140312702207168 spec.py:363] Evaluating on the test split. +I0915 09:14:11.029709 140312702207168 submission_runner.py:516] Time since start: 6864.70s, Step: 28161, {'train/accuracy': 0.9938759347873749, 'train/loss': 0.019595363183518063, 'train/mean_average_precision': 0.6564537381256985, 'validation/accuracy': 0.9867420982727099, 'validation/loss': 0.047822139312696674, 'validation/mean_average_precision': 0.27968567955907997, 'validation/num_examples': 43793, 'test/accuracy': 0.9859790236001345, 'test/loss': 0.050509282329632026, 'test/mean_average_precision': 0.2693004772225223, 'test/num_examples': 43793, 'score': 6355.462056875229, 'total_duration': 6864.697921037674, 'accumulated_submission_time': 6355.462056875229, 'accumulated_eval_time': 489.1870641708374, 'accumulated_logging_time': 0.47249269485473633} +I0915 09:14:11.055424 140277943293696 logging_writer.py:48] [28161] accumulated_eval_time=489.187, accumulated_logging_time=0.472493, accumulated_submission_time=6355.46, global_step=28161, preemption_count=0, score=6355.46, test/accuracy=0.985979, test/loss=0.0505093, test/mean_average_precision=0.2693, test/num_examples=43793, total_duration=6864.7, train/accuracy=0.993876, train/loss=0.0195954, train/mean_average_precision=0.656454, validation/accuracy=0.986742, validation/loss=0.0478221, validation/mean_average_precision=0.279686, validation/num_examples=43793 +I0915 09:15:25.731505 140277951686400 logging_writer.py:48] [28500] global_step=28500, grad_norm=0.023367, loss=0.0232377 +I0915 09:15:25.734611 140312702207168 submission.py:307] 28500) loss = 0.023, grad_norm = 0.023 +I0915 09:17:15.443037 140277943293696 logging_writer.py:48] [29000] global_step=29000, grad_norm=0.0243113, loss=0.0247532 +I0915 09:17:15.446342 140312702207168 submission.py:307] 29000) loss = 0.025, grad_norm = 0.024 +I0915 09:19:05.451516 140277951686400 logging_writer.py:48] [29500] global_step=29500, grad_norm=0.0237403, loss=0.0248026 +I0915 09:19:05.454743 140312702207168 submission.py:307] 29500) loss = 0.025, grad_norm = 0.024 +I0915 09:20:54.801815 140277943293696 logging_writer.py:48] [30000] global_step=30000, grad_norm=0.0235136, loss=0.023279 +I0915 09:20:54.804871 140312702207168 submission.py:307] 30000) loss = 0.023, grad_norm = 0.024 +I0915 09:21:43.616653 140312702207168 spec.py:333] Evaluating on the training split. +I0915 09:22:08.506850 140312702207168 spec.py:346] Evaluating on the validation split. +I0915 09:22:10.529337 140312702207168 spec.py:363] Evaluating on the test split. +I0915 09:22:12.562218 140312702207168 submission_runner.py:516] Time since start: 7346.23s, Step: 30223, {'train/accuracy': 0.994096175029288, 'train/loss': 0.018770395171369213, 'train/mean_average_precision': 0.6797509196241163, 'validation/accuracy': 0.9866631459477364, 'validation/loss': 0.0483967857676358, 'validation/mean_average_precision': 0.27737531200076043, 'validation/num_examples': 43793, 'test/accuracy': 0.9859232218618155, 'test/loss': 0.05131975322576162, 'test/mean_average_precision': 0.26714178063653554, 'test/num_examples': 43793, 'score': 6806.630153417587, 'total_duration': 7346.230459213257, 'accumulated_submission_time': 6806.630153417587, 'accumulated_eval_time': 518.1325941085815, 'accumulated_logging_time': 0.5071706771850586} +I0915 09:22:12.587544 140277951686400 logging_writer.py:48] [30223] accumulated_eval_time=518.133, accumulated_logging_time=0.507171, accumulated_submission_time=6806.63, global_step=30223, preemption_count=0, score=6806.63, test/accuracy=0.985923, test/loss=0.0513198, test/mean_average_precision=0.267142, test/num_examples=43793, total_duration=7346.23, train/accuracy=0.994096, train/loss=0.0187704, train/mean_average_precision=0.679751, validation/accuracy=0.986663, validation/loss=0.0483968, validation/mean_average_precision=0.277375, validation/num_examples=43793 +I0915 09:23:13.679830 140277943293696 logging_writer.py:48] [30500] global_step=30500, grad_norm=0.0257483, loss=0.0246587 +I0915 09:23:13.683124 140312702207168 submission.py:307] 30500) loss = 0.025, grad_norm = 0.026 +I0915 09:25:03.286993 140277951686400 logging_writer.py:48] [31000] global_step=31000, grad_norm=0.0244895, loss=0.0216894 +I0915 09:25:03.289979 140312702207168 submission.py:307] 31000) loss = 0.022, grad_norm = 0.024 +I0915 09:26:52.741905 140277943293696 logging_writer.py:48] [31500] global_step=31500, grad_norm=0.0250382, loss=0.0232266 +I0915 09:26:52.745398 140312702207168 submission.py:307] 31500) loss = 0.023, grad_norm = 0.025 +I0915 09:28:42.116299 140277951686400 logging_writer.py:48] [32000] global_step=32000, grad_norm=0.0265271, loss=0.0232666 +I0915 09:28:42.119640 140312702207168 submission.py:307] 32000) loss = 0.023, grad_norm = 0.027 +I0915 09:29:45.025561 140312702207168 spec.py:333] Evaluating on the training split. +I0915 09:30:09.879477 140312702207168 spec.py:346] Evaluating on the validation split. +I0915 09:30:11.846281 140312702207168 spec.py:363] Evaluating on the test split. +I0915 09:30:13.867090 140312702207168 submission_runner.py:516] Time since start: 7827.54s, Step: 32287, {'train/accuracy': 0.9941125453466377, 'train/loss': 0.01840711206501748, 'train/mean_average_precision': 0.6897578595504239, 'validation/accuracy': 0.9867088124596105, 'validation/loss': 0.049235508341343576, 'validation/mean_average_precision': 0.28021762684262463, 'validation/num_examples': 43793, 'test/accuracy': 0.9859206949906464, 'test/loss': 0.05211511224782878, 'test/mean_average_precision': 0.27237371187706816, 'test/num_examples': 43793, 'score': 7257.683073759079, 'total_duration': 7827.535281896591, 'accumulated_submission_time': 7257.683073759079, 'accumulated_eval_time': 546.9740028381348, 'accumulated_logging_time': 0.5415818691253662} +I0915 09:30:13.895424 140277943293696 logging_writer.py:48] [32287] accumulated_eval_time=546.974, accumulated_logging_time=0.541582, accumulated_submission_time=7257.68, global_step=32287, preemption_count=0, score=7257.68, test/accuracy=0.985921, test/loss=0.0521151, test/mean_average_precision=0.272374, test/num_examples=43793, total_duration=7827.54, train/accuracy=0.994113, train/loss=0.0184071, train/mean_average_precision=0.689758, validation/accuracy=0.986709, validation/loss=0.0492355, validation/mean_average_precision=0.280218, validation/num_examples=43793 +I0915 09:31:00.961650 140277951686400 logging_writer.py:48] [32500] global_step=32500, grad_norm=0.0254064, loss=0.0233024 +I0915 09:31:00.964497 140312702207168 submission.py:307] 32500) loss = 0.023, grad_norm = 0.025 +I0915 09:32:49.490809 140277943293696 logging_writer.py:48] [33000] global_step=33000, grad_norm=0.0272416, loss=0.0237096 +I0915 09:32:49.494205 140312702207168 submission.py:307] 33000) loss = 0.024, grad_norm = 0.027 +I0915 09:34:39.025581 140277951686400 logging_writer.py:48] [33500] global_step=33500, grad_norm=0.022787, loss=0.0218786 +I0915 09:34:39.028716 140312702207168 submission.py:307] 33500) loss = 0.022, grad_norm = 0.023 +I0915 09:36:28.276791 140277943293696 logging_writer.py:48] [34000] global_step=34000, grad_norm=0.0221849, loss=0.0198135 +I0915 09:36:28.280045 140312702207168 submission.py:307] 34000) loss = 0.020, grad_norm = 0.022 +I0915 09:37:46.485064 140312702207168 spec.py:333] Evaluating on the training split. +I0915 09:38:11.296739 140312702207168 spec.py:346] Evaluating on the validation split. +I0915 09:38:13.266875 140312702207168 spec.py:363] Evaluating on the test split. +I0915 09:38:15.234013 140312702207168 submission_runner.py:516] Time since start: 8308.90s, Step: 34360, {'train/accuracy': 0.9943937974800934, 'train/loss': 0.017529049207899678, 'train/mean_average_precision': 0.6996457377273805, 'validation/accuracy': 0.9866365578897119, 'validation/loss': 0.050051150298841655, 'validation/mean_average_precision': 0.27472048844387825, 'validation/num_examples': 43793, 'test/accuracy': 0.9859072183444109, 'test/loss': 0.05294910805132749, 'test/mean_average_precision': 0.2711870785630257, 'test/num_examples': 43793, 'score': 7708.8865878582, 'total_duration': 8308.902286291122, 'accumulated_submission_time': 7708.8865878582, 'accumulated_eval_time': 575.7229630947113, 'accumulated_logging_time': 0.5790755748748779} +I0915 09:38:15.259997 140277951686400 logging_writer.py:48] [34360] accumulated_eval_time=575.723, accumulated_logging_time=0.579076, accumulated_submission_time=7708.89, global_step=34360, preemption_count=0, score=7708.89, test/accuracy=0.985907, test/loss=0.0529491, test/mean_average_precision=0.271187, test/num_examples=43793, total_duration=8308.9, train/accuracy=0.994394, train/loss=0.017529, train/mean_average_precision=0.699646, validation/accuracy=0.986637, validation/loss=0.0500512, validation/mean_average_precision=0.27472, validation/num_examples=43793 +I0915 09:38:46.437846 140277943293696 logging_writer.py:48] [34500] global_step=34500, grad_norm=0.0240877, loss=0.017965 +I0915 09:38:46.441085 140312702207168 submission.py:307] 34500) loss = 0.018, grad_norm = 0.024 +I0915 09:40:35.296383 140277951686400 logging_writer.py:48] [35000] global_step=35000, grad_norm=0.0280715, loss=0.021607 +I0915 09:40:35.299681 140312702207168 submission.py:307] 35000) loss = 0.022, grad_norm = 0.028 +I0915 09:42:25.166929 140277943293696 logging_writer.py:48] [35500] global_step=35500, grad_norm=0.0256688, loss=0.0206609 +I0915 09:42:25.170141 140312702207168 submission.py:307] 35500) loss = 0.021, grad_norm = 0.026 +I0915 09:44:14.260565 140277951686400 logging_writer.py:48] [36000] global_step=36000, grad_norm=0.0258043, loss=0.0205732 +I0915 09:44:14.263663 140312702207168 submission.py:307] 36000) loss = 0.021, grad_norm = 0.026 +I0915 09:45:47.650881 140312702207168 spec.py:333] Evaluating on the training split. +I0915 09:46:12.257864 140312702207168 spec.py:346] Evaluating on the validation split. +I0915 09:46:14.229900 140312702207168 spec.py:363] Evaluating on the test split. +I0915 09:46:16.254717 140312702207168 submission_runner.py:516] Time since start: 8789.92s, Step: 36429, {'train/accuracy': 0.9946867698661335, 'train/loss': 0.01641026124824217, 'train/mean_average_precision': 0.7309813972844874, 'validation/accuracy': 0.9866442704561618, 'validation/loss': 0.05097477770049425, 'validation/mean_average_precision': 0.27710463975447336, 'validation/num_examples': 43793, 'test/accuracy': 0.9858187778534904, 'test/loss': 0.05411188335394137, 'test/mean_average_precision': 0.26892242368784797, 'test/num_examples': 43793, 'score': 8159.898387432098, 'total_duration': 8789.92292046547, 'accumulated_submission_time': 8159.898387432098, 'accumulated_eval_time': 604.3266940116882, 'accumulated_logging_time': 0.6139218807220459} +I0915 09:46:16.280272 140277943293696 logging_writer.py:48] [36429] accumulated_eval_time=604.327, accumulated_logging_time=0.613922, accumulated_submission_time=8159.9, global_step=36429, preemption_count=0, score=8159.9, test/accuracy=0.985819, test/loss=0.0541119, test/mean_average_precision=0.268922, test/num_examples=43793, total_duration=8789.92, train/accuracy=0.994687, train/loss=0.0164103, train/mean_average_precision=0.730981, validation/accuracy=0.986644, validation/loss=0.0509748, validation/mean_average_precision=0.277105, validation/num_examples=43793 +I0915 09:46:32.633668 140277951686400 logging_writer.py:48] [36500] global_step=36500, grad_norm=0.0257267, loss=0.020047 +I0915 09:46:32.636606 140312702207168 submission.py:307] 36500) loss = 0.020, grad_norm = 0.026 +I0915 09:48:21.585793 140277943293696 logging_writer.py:48] [37000] global_step=37000, grad_norm=0.0276746, loss=0.0195555 +I0915 09:48:21.588962 140312702207168 submission.py:307] 37000) loss = 0.020, grad_norm = 0.028 +I0915 09:50:11.523927 140277951686400 logging_writer.py:48] [37500] global_step=37500, grad_norm=0.0238782, loss=0.0173211 +I0915 09:50:11.528090 140312702207168 submission.py:307] 37500) loss = 0.017, grad_norm = 0.024 +I0915 09:52:00.118777 140277943293696 logging_writer.py:48] [38000] global_step=38000, grad_norm=0.0267336, loss=0.0206242 +I0915 09:52:00.121922 140312702207168 submission.py:307] 38000) loss = 0.021, grad_norm = 0.027 +I0915 09:53:48.819908 140312702207168 spec.py:333] Evaluating on the training split. +I0915 09:54:13.588750 140312702207168 spec.py:346] Evaluating on the validation split. +I0915 09:54:15.537718 140312702207168 spec.py:363] Evaluating on the test split. +I0915 09:54:17.492845 140312702207168 submission_runner.py:516] Time since start: 9271.16s, Step: 38499, {'train/accuracy': 0.995182106982942, 'train/loss': 0.01506322610779344, 'train/mean_average_precision': 0.7713066507784985, 'validation/accuracy': 0.9865285819594141, 'validation/loss': 0.051809748643400155, 'validation/mean_average_precision': 0.2780833643058275, 'validation/num_examples': 43793, 'test/accuracy': 0.9857585540906254, 'test/loss': 0.055032536361676127, 'test/mean_average_precision': 0.26985846816098746, 'test/num_examples': 43793, 'score': 8611.054082632065, 'total_duration': 9271.161074876785, 'accumulated_submission_time': 8611.054082632065, 'accumulated_eval_time': 632.9995555877686, 'accumulated_logging_time': 0.6483712196350098} +I0915 09:54:17.518290 140277951686400 logging_writer.py:48] [38499] accumulated_eval_time=633, accumulated_logging_time=0.648371, accumulated_submission_time=8611.05, global_step=38499, preemption_count=0, score=8611.05, test/accuracy=0.985759, test/loss=0.0550325, test/mean_average_precision=0.269858, test/num_examples=43793, total_duration=9271.16, train/accuracy=0.995182, train/loss=0.0150632, train/mean_average_precision=0.771307, validation/accuracy=0.986529, validation/loss=0.0518097, validation/mean_average_precision=0.278083, validation/num_examples=43793 +I0915 09:54:18.419856 140277943293696 logging_writer.py:48] [38500] global_step=38500, grad_norm=0.0296274, loss=0.020145 +I0915 09:54:18.422708 140312702207168 submission.py:307] 38500) loss = 0.020, grad_norm = 0.030 +I0915 09:56:07.757311 140277951686400 logging_writer.py:48] [39000] global_step=39000, grad_norm=0.0267548, loss=0.0206569 +I0915 09:56:07.760574 140312702207168 submission.py:307] 39000) loss = 0.021, grad_norm = 0.027 +I0915 09:57:57.425104 140277943293696 logging_writer.py:48] [39500] global_step=39500, grad_norm=0.0288708, loss=0.0192192 +I0915 09:57:57.428385 140312702207168 submission.py:307] 39500) loss = 0.019, grad_norm = 0.029 +I0915 09:59:46.613585 140277951686400 logging_writer.py:48] [40000] global_step=40000, grad_norm=0.02897, loss=0.0214014 +I0915 09:59:46.616600 140312702207168 submission.py:307] 40000) loss = 0.021, grad_norm = 0.029 +I0915 10:01:35.025608 140277943293696 logging_writer.py:48] [40500] global_step=40500, grad_norm=0.026218, loss=0.019323 +I0915 10:01:35.028786 140312702207168 submission.py:307] 40500) loss = 0.019, grad_norm = 0.026 +I0915 10:01:49.985388 140312702207168 spec.py:333] Evaluating on the training split. +I0915 10:02:14.715881 140312702207168 spec.py:346] Evaluating on the validation split. +I0915 10:02:16.690908 140312702207168 spec.py:363] Evaluating on the test split. +I0915 10:02:18.731091 140312702207168 submission_runner.py:516] Time since start: 9752.40s, Step: 40568, {'train/accuracy': 0.9954051832302168, 'train/loss': 0.014298583810567056, 'train/mean_average_precision': 0.7786078554565572, 'validation/accuracy': 0.9865267552989391, 'validation/loss': 0.05242167355994207, 'validation/mean_average_precision': 0.28183505269730574, 'validation/num_examples': 43793, 'test/accuracy': 0.9857678192849123, 'test/loss': 0.05568107364018534, 'test/mean_average_precision': 0.27240996826805103, 'test/num_examples': 43793, 'score': 9062.13857460022, 'total_duration': 9752.399313926697, 'accumulated_submission_time': 9062.13857460022, 'accumulated_eval_time': 661.7452442646027, 'accumulated_logging_time': 0.6828422546386719} +I0915 10:02:18.755366 140277951686400 logging_writer.py:48] [40568] accumulated_eval_time=661.745, accumulated_logging_time=0.682842, accumulated_submission_time=9062.14, global_step=40568, preemption_count=0, score=9062.14, test/accuracy=0.985768, test/loss=0.0556811, test/mean_average_precision=0.27241, test/num_examples=43793, total_duration=9752.4, train/accuracy=0.995405, train/loss=0.0142986, train/mean_average_precision=0.778608, validation/accuracy=0.986527, validation/loss=0.0524217, validation/mean_average_precision=0.281835, validation/num_examples=43793 +I0915 10:02:19.201720 140277943293696 logging_writer.py:48] [40568] global_step=40568, preemption_count=0, score=9062.14 +I0915 10:02:19.301688 140312702207168 submission_runner.py:857] Final ogbg score: 9062.13857460022 diff --git a/logs/self_tuning/ademamix_golden/study_2/ogbg_pytorch/trial_1/eval_measurements.csv b/logs/self_tuning/ademamix_golden/study_2/ogbg_pytorch/trial_1/eval_measurements.csv new file mode 100644 index 00000000..c9d7f088 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/ogbg_pytorch/trial_1/eval_measurements.csv @@ -0,0 +1,22 @@ +accumulated_eval_time,accumulated_logging_time,accumulated_submission_time,global_step,preemption_count,score,test/accuracy,test/loss,test/mean_average_precision,test/num_examples,total_duration,train/accuracy,train/loss,train/mean_average_precision,validation/accuracy,validation/loss,validation/mean_average_precision,validation/num_examples +80.14644813537598,0.0,39.48756432533264,1,0,39.48756432533264,0.4335950891101118,0.8399119195882211,0.0299246871300735,43793,120.05303835868835,0.4288207246897473,0.8422619610364236,0.0249510035387999,0.4298933798576991,0.8421632409340811,0.0290027249317241,43793 +110.27350735664368,0.0300910472869873,490.7094373703003,1936,0,490.7094373703003,0.983686098586721,0.0584451219383797,0.106758292979182,43793,602.7956111431122,0.9874387078307416,0.046041251289779,0.1099181501609003,0.9846562549725756,0.0552443523717359,0.1103213394517892,43793 +140.7103328704834,0.0647165775299072,941.8389570713044,3864,0,941.8389570713044,0.9845300735572198,0.0531398670834179,0.1678125760208756,43793,1085.7501618862152,0.9882669022075652,0.0408082194156488,0.1890120243562696,0.9853883398984864,0.0503943785335732,0.162372326140947,43793 +170.97252202033997,0.099391222000122,1392.9659972190857,5794,0,1392.9659972190857,0.98500933678897,0.0500873639385161,0.201150206811026,43793,1568.5372149944303,0.989055488586244,0.0373807219879116,0.2414374545903787,0.9858494701872772,0.0474400850290154,0.1987696427509348,43793 +200.6434156894684,0.1341695785522461,1844.2267653942108,7737,0,1844.2267653942108,0.9855458757672212,0.0483984236482713,0.2330750212756477,43793,2050.8582327365875,0.9896002426085612,0.0351999669500902,0.2977852960696569,0.9863487573837676,0.0457522570927196,0.2323031181715506,43793 +229.55694007873527,0.1670575141906738,2295.3961102962494,9754,0,2295.3961102962494,0.9856402122908698,0.0477264864898727,0.2528231426107782,43793,2532.3440577983856,0.990322180217352,0.032537894384189,0.353039035700779,0.9864611984841154,0.0452175720069559,0.2438146719936962,43793 +258.4821677207947,0.1989660263061523,2746.4137926101685,11792,0,2746.4137926101685,0.9859383830888304,0.0470001525072036,0.2628282831636894,43793,3013.6949014663696,0.9906539829082384,0.0311304128096638,0.3894310935237193,0.9867177427997104,0.04426489880098,0.2625295256044753,43793 +287.21206617355347,0.2319512367248535,3197.621773481369,13827,0,3197.621773481369,0.985886161084668,0.047356559789141,0.253993981295005,43793,3495.029235601425,0.990904449564144,0.0300627034537495,0.4085010932406117,0.9865992128311126,0.0448302121625549,0.2585199051539353,43793 +316.0617251396179,0.2747120857238769,3648.6258997917175,15863,0,3648.6258997917175,0.986044511677935,0.0471183034755428,0.267966429826295,43793,3976.294326543808,0.9915098901523316,0.0278687613131511,0.4657407301065514,0.986842361636558,0.0445236503313156,0.2741214480682246,43793 +344.84174847602844,0.308274507522583,4099.839018344879,17900,0,4099.839018344879,0.98603019274131,0.0473273395510508,0.2693112419550987,43793,4457.6909737586975,0.9922223546765736,0.0256195144504969,0.5099060104916786,0.9867916210678088,0.044740325245016,0.2766751670939729,43793 +373.768721818924,0.340721845626831,4550.96763586998,19944,0,4550.96763586998,0.9861186332322304,0.0476906134737824,0.2669362705916213,43793,4939.150961399078,0.9924673468409008,0.0245501953872818,0.5369672115837776,0.9868094817480084,0.0452432277070296,0.2700441549799042,43793 +402.4578549861908,0.3742103576660156,5002.144770145416,21971,0,5002.144770145416,0.9859952376901364,0.0482437975578211,0.2705550973973591,43793,5420.412091016769,0.9925652639004054,0.0239635293389369,0.5399600387924857,0.9868918844316568,0.0454523866739841,0.2773576826036955,43793 +431.29218077659607,0.4074556827545166,5453.354311943054,24028,0,5453.354311943054,0.9860739918415752,0.0489102631188833,0.2715814145383403,43793,5901.869510173798,0.9929287971617616,0.0228626902871778,0.5828592682832358,0.9867441278954598,0.046245322226967,0.2789765513309469,43793 +460.3590455055237,0.4385535717010498,5904.460551261902,26090,0,5904.460551261902,0.985984709060265,0.0498072793525566,0.2677533639503699,43793,6383.45326423645,0.9933762598810584,0.0210874283054507,0.616506450120422,0.9867473752918596,0.0469827000284553,0.2799395559942953,43793 +489.1870641708374,0.4724926948547363,6355.462056875229,28161,0,6355.462056875229,0.9859790236001345,0.050509282329632,0.2693004772225223,43793,6864.697921037674,0.9938759347873748,0.019595363183518,0.6564537381256985,0.98674209827271,0.0478221393126966,0.2796856795590799,43793 +518.1325941085815,0.5071706771850586,6806.630153417587,30223,0,6806.630153417587,0.9859232218618156,0.0513197532257616,0.2671417806365355,43793,7346.230459213257,0.994096175029288,0.0187703951713692,0.6797509196241163,0.9866631459477364,0.0483967857676358,0.2773753120007604,43793 +546.9740028381348,0.5415818691253662,7257.683073759079,32287,0,7257.683073759079,0.9859206949906464,0.0521151122478287,0.2723737118770681,43793,7827.535281896591,0.9941125453466376,0.0184071120650174,0.6897578595504239,0.9867088124596104,0.0492355083413435,0.2802176268426246,43793 +575.7229630947113,0.5790755748748779,7708.8865878582,34360,0,7708.8865878582,0.9859072183444108,0.0529491080513274,0.2711870785630257,43793,8308.902286291122,0.9943937974800934,0.0175290492078996,0.6996457377273805,0.986636557889712,0.0500511502988416,0.2747204884438782,43793 +604.3266940116882,0.6139218807220459,8159.898387432098,36429,0,8159.898387432098,0.9858187778534904,0.0541118833539413,0.2689224236878479,43793,8789.92292046547,0.9946867698661336,0.0164102612482421,0.7309813972844874,0.9866442704561618,0.0509747777004942,0.2771046397544733,43793 +632.9995555877686,0.6483712196350098,8611.054082632065,38499,0,8611.054082632065,0.9857585540906254,0.0550325363616761,0.2698584681609874,43793,9271.161074876783,0.995182106982942,0.0150632261077934,0.7713066507784985,0.986528581959414,0.0518097486434001,0.2780833643058275,43793 +661.7452442646027,0.6828422546386719,9062.13857460022,40568,0,9062.13857460022,0.9857678192849123,0.05568107364018534,0.27240996826805103,43793,9752.399313926697,0.9954051832302168,0.014298583810567056,0.7786078554565572,0.9865267552989391,0.05242167355994207,0.28183505269730574,43793 diff --git a/logs/self_tuning/ademamix_golden/study_2/ogbg_pytorch/trial_1/events.out.tfevents.1789456786.d8fa15bfabc6.38.0.v2 b/logs/self_tuning/ademamix_golden/study_2/ogbg_pytorch/trial_1/events.out.tfevents.1789456786.d8fa15bfabc6.38.0.v2 new file mode 100644 index 0000000000000000000000000000000000000000..c7c12729a3e89632976f24075d812bce5d554259 GIT binary patch literal 66056 zcmbWAc|c8D8^%jf=ZKOhNo7h(rZSZhsm#+wN+=1Xc`~NVAqk-rLP!}xDHU=>h{{+I zWe!OQ$@E?4^zFT#J=BSR@Ap36`n_wfwSH%xeRgJ2;(x`r&0h(mq=M%RPIxd>y~S)F zM;DKtmI2Nle#U(#brE**bN2A{^qK4Cxv-~`r;l?_e?J$up3Z;%yYH-pJ}!RFJ}27C z-R)7AR(<@2rQlwkug9M>y^caHp{$#yukSFSytJ}ZC)ruUrw$p_TqqFA`8qkeIr{ic zud3PEr>=JQs#UKA`Neg$I>IhO!8{+wIkP-Gecb;aQ1kn@83fcydnp;#1t=WWo+@~4 z?I->}*A?mro6mFeobBi~%h%7@>;Dmd{>Kn;ZU54QfprlF76;l0Y=pa3{u#fLw4T1O zyRfC-pPuw`baL|dadcYz|C~8t4{<1`mK=08s>_*asA@0R*P6=dChRH{igU<*EZt*+ z{Wx{tPzS5J44-~y4+`wtQyGJV1B4pl40mTok6DfZ&OVOwoM(CYI6Jxcx_Em0e`E@C zZwv;)dBI0KP3uw}Mt-Oe?CM0N_7?UOwhnM~bD87l_g@AJ^M`r~Vp{2&=ntw3+bB2S zgTPmh3hN=%6Dks6B&$p$gH>ztoMWTwg4PM5b_mw>p@PhWgM}T5p!$KecD_U4i}6!B z46TcnI`VI#z`~e{HWKy~suIy2{_eA!7dg6nxj9qUsZ=X{Yv^F{&||*`)CImOziunA z8cGG~33Y`n{G5ILSiLildW^j@zo&TxuGX<5s2I@&FNz_%wn&!IwVFKPE6ci(ynQ)8 zN#Hu3iWn&DFKjQ0sJ~x(x_ridP13KmF{ukx?|jr=;69ZK?J4XbY$Xbns8eyBnosXl z7v(;q!vuliEGnv#P+Qp4_s=%+|4ViFrw+rQ(N%-I_rPNO;7pZ^7o(@&e+zofe@pSt z<@)~|-S$-%}>ae=F#h*M91U5^lxPC%Ip&A+I z<~eVki^n`j{8J0vo4qIJYrne0`@$4U?=|FL{vRq6d!=^YojX^PMlUgKD4E#z} zb5pS9V)+eiXP}V!O-1lim?K5%^!=xRVo_R+m_n9Kk(@Ibuc&>SR?P;A*et4ALW-y` zgzP5L@7|!+!U<2ShirMEKsadp(6Mxq=v#C_0x3yPM~=F z!dgrrU{g$=HUTM|(h^$&#rF49wS*Ks1LHmbMfz7Nf}cVODRS=IEddJC**0PdA)BJ- zj`2uw=6W{;pjhQfRdZ9YuBaHl{TG2^i4PUQPoa$z-yZdH0t#27d18v@Yzn1w)rZ7w$jTGSuL#G49 z$`@2MHwEkSskfss?zCp_s0e-vbEG(!{$n1@h2Y3qF+~eDg|_}EqzJs>`y43DPEys} z6s*-9@^t<$n2VWbsR(|G2}sf9<=yu{G2efOn4%?{;?&F$NRjwbVH{A{1yR)!QgrUS zvl=LZ)=?4s6b?u+cl(-;Kv6#CkeEV|O>xxP3Mpnix6cQPn#oi(HwDY1+6?%O=S8tS z6~Ry8g%m56pYR2Wes5F66iRH0TfW1QqU7GMfj}|o6ICrCMV?cCEtm`Q?^FaoMKDsB z^o`K~ikc0V#T2dB6hChaMGB)-U%Yixxs9skreLiL=P8{ufMQV$6~Rvtf)v-Lm#hPd z&SUP0DU{h1N$TcEF*tnRE0~MR|4`N36s(cMMwP>p+F&}NmX-Gu;wEASZN|qbkU(A_$eZh!msJlsX#IHUbUD)g-y{p z&;%)LEUNbaMNlbK%}v2NFD?}YSpr4$D=LDYA{HqYou0HBC=|kFSBpMhv|&?}Eggsy zEgbds0mY+8s+ybP3-jzYT~_1)6os)=1V2SQQkbrFdI%JsG*rYCZP^q_dHs;0|L5)z zKyj}NRn1MoS{FkKFQ))SLJul}pCSn3>4;TI*Tb(*%baK43R?K=SMbB4B13g zOGt4e&F>vh9N11p@KdBA#me!5tw2$rU?`?gV^b)Z^g)XKel4Sb;*1Ja%}v3&qR!2o z=>Qb+9jFL?iY%m1i90tAC=w%wi7DE#DHc}tM2dKiuB(7z%6_U^LW+N!(%%9_yF@C2 zpW-S~1n;>!4Jf4BOcGP5vnl+x_dp60m5c#Eu}y=jmXKmw{I+PI=+l*o;HM}+ijvi9 zLV@B{qNA9iJ)0tCp&n8g?^H^-+ZH=jfgykCHc}Mq zlMjQrkgKAqxhXy}DWo=hH3f>6HBsfiDGVz#k>c`?q5eQI zB8#f#reIxB-v&JU3>5NLsR(`wDg1(F<+t-zKoPz!OH9#`O;K?|11WZt-!cFSg)LOI zgcRnb2DnGrue{I7k+Oa`T>Re z04jo?LIEk}I%ReTiZ#uv#1viF6bFObAVs;);{!m^tu<9GAw|}hoA-cXOnWMVpF#;K zlA;|p0Y#X!)Ed#}i>_>nyyvZvLd`ec4Jb|sscH!+^0E^<0LAClR0KbT3R2wJcxfY0 zd@olNQ|PcMV!yURiW!{`eFTcs4^%Zb#e3!z)g$5+-UG3%r6Tw#)Q}?7r-uwsJWbLP zQ*>if^gE=86mhPv@J`N)vs5)VMJ1EMQ#xo6%th5jDuSOv11Tz-d|3+=!w>foQ|PiO zmUnA`6c2xo!27hxC#Y&}iV7x0WzRF;fnrh`6~RxTjTDndpI!+R^4o@pDY~;M`p#>P z6c-$uw1z9HNi(3its&DH8;g;<`p$6G-Mf2JUdK9 z@KYEeMbqOyoq^)1-DWXGPc}u%v?fUL;Y4i^QaDr9+!Sw^6c2kj&IO79cPfIPVjxm< z9=7ZvP<%JpE2ikhrqF98jTAn|6Lf(h!;-4zreMv5=k_6~Kyhp=6~Rw27%2|7--XYC z7;ByoQ}kw2tTvQFiYvp{;l07DI#e|`#Vh7q4D(Sm1d9BgR0KbTIa1t|*Rp`QSR=e3 zrs%__=se&jK1}$rN2{kW7q%)?wS*LBW1de2isc$q1V6=aq*#=$eibN6KIMri`m!my znS4WvtG0J>b*ulNs<|m%GUuYoc9j4q*2?Dpe*pXxBaq@+w$=!s82{$6IK+@mF{|`5 zQjFer1@D0bf269pDOgujn~9t7&VNcR6~Rw21}Q33JDC8**pf;yg%O+L{_syou{-*# zGF(yLUsBcF6fc-_vBWp$Ay~36;f!94N?b+ zoD8a(n}T&kIlQ$A1&YW^R0Kc8WTa?vy5A?5i#U&#Yek=u`m-q_JSveQbnup@K#>+e zRdZ9Y<|6Csn#Dkox}1vOrEkiZ;`^{TaR*}P;;QFxhbA8 zuc#vjW4wT3wXOJ;f9Q*%TGM%8^3ugFD`*b+V$WC8SvQ(eeUN zn2)0(_$eHaVwT#jEkH5Sz)Varh)t2H`4TBwm`v>mbK%^ds^+G6!ki2Fl}+{nMVJ{C z!A~&*3_$lTi z#c=ywr-5RBqi- zmrm|LkMd>BR4_bCkUW?Bj^i!a!xhWnqDc(Ca!6%EFs!$R96n;oCI%J7o&5jYHo@)a=-AROqnX}26Lf$hl=2*ScDXLbM5v6Me)oS zF~txzg{JKT9D?=>rvRWxpGQ@5Qykm<6(bI+^~LARmv~YU{1nTO!lZxlP@u?FJSwIz zXH(=W7UB>Fv`fV&v{TfmYHo_d!3?2%X5Ysb=AvUqDuSOP7%6P}cTE6_pvM_v3Xb3A zDG$4g6y{cT-C!;q94}E3{1hQb@vEX*1?HmtzVBiROE!h2dL9nJzu))% zKr!YBRn1M2GUzo%td1F11{7{5sR(|G4M?#jqAz+>>H6mDM4vCL*c72|xj2MZS+15q zkr+W$b5k^b@d_hUy18}%3e(+G1V2S6Qk2U-ybE)&JgmK#!kSI--SY+xq0RldzCcmD zgR17HXttIy7l$0w@zndahl=2*2t$eyu7kpWqSJbPF~tZr#hn+|kz%KyA{r>(Y@w>T zDfX2yC}J<1#Bao&L{Sm^6yZqmXhH8DK%q6#L`*S~O(Cs%6)C0#xcvr-DHEw`Zi?|c z86ULPFCFj?Tu}!8T|1n^+m|_&0qL*3@QgoZ|y&fnYnN!u=6iEq; zk0`ob67B%SsS#8JKScymSk0VW26N#)aJracG@HU=-W8;9vD@7 zNdZr&2!4uPNO4>GIX>Im_vi*O#W*&_@l*dIMYEd6c=sSRnX2Zdcz5qUMg*VPhR+`4 zWKt3Q6fsCKp>GsEdvH8$mzZKan?k+aMI1uFbomW17gbqQH8;f>X@-iO-#T;)Tv1=I zQW5+Vdypb9un14R@^cAdiV19rt+&r3#f*0*kBY>MQu7jOtV!#B4Dio++UYHo_oUilbN{q4&Wpm>o=MetMX zM~W>O3b@neZMrU|n8cAA*X0R0~O1-EEeu_g#ad+Wq zynB#n_C`!GnN86ol0h-U{Anyuw6>WFevOl-kl8;n&YVmeu{Xcco%MP6XxRL zfLbxd6gGwSh;*Eb)5^gQfa2p2s+yZZn0E~$IxV~t0~8;usR(|G1f&>Tatfc8w^I^^ zh(2FTWmEL+$)K1Mr?nO+PPU_}xheK6sw=N!B4D*W!1$L zHf#!?FK4llIRP?w_h5Sqs+yZ3@+U*ZR%m`G0*WJTs0egM4!eh%3S~Y8=yG( zgsSGI&^*VG`}tjN#R5g*Ybt`DA`K}<*i_=vESK&t5>w1zQ|$3$P+WQ42j8Tz?Fm)Q zP4TP+gJSt6tuUb2`Iy_ox-`e?VY?G=HhfDRn1ND zNiho}7CNpQ4ix$Os0e`QfBgU` zR*j<~_$mHH3dMs?_>`AJ)2m{N*=&mM!;a$>b#vLgmO$~Y1y#*Wv1|Z?BDwRiPC#+2 z4Hdypk&P5noxFo!U5u+N6jL~{DcWp4ffNUiJe~y<2ft9&+!V3DGBIM#vJT6E;@mGP zf}i3FQrtKGaTX}P#l8|#%wbcgPCJG}(EZv6pZQ;TgsSGIxLVHm=w9(*3;x#L>m(Jy zPjM9~F1Q{23h&t4J!gEZ$jqxbUgmW#aJ4DCW+(N#zZATv@}WCj&dpKn*c3Bb%{MUb zU-tmRc$?7og02hc41SPW3mA#uoNF|Is5;x86M>X-ynV^adNdBgKla#60IBh$%eg_? zpIU&gK2#0bvIjtRETS{`L6!^$5c}{Pc>tO9*7|`6q=e&jIhAe4Ajl*=X$JuDt)|Pl zL5#ZkW5#@$bIk!n>L;DS3lf}xAop#2I>6gKPZf{+E&>TxtnY28SFbL^XKWpX`rTo< z_IyqU@w^T-Yx`hivbcD6p>^F=mVW7bv_PeT4w3kMsM#7%Fjn=EFvP4bR`mYc?Vsq_ zhF*y3smvf%zd52iyf5)*EuB{18&O|b&&F|8gbvs@rfyuH@9vKkC^Q*GHNSDMMAfZg zFe@u$RKm+90|j*UU%eCcaL?)Y;8mUBN^~OsOHm)(UW!-BPq)3EaPd2;(K(`vpY>MM zZgV`bbKXOqD8YMk*J{xjlD-yIE$#uP&dDF?GPrI&uBINCB-pP*r#AAvsNVUv1A5nG zZU0;_OYqr%&SJk9HQ(+tZk2{UANIq-%9!v*G`f60x z@P4OQ*_zkk>8e(57dTnasSUpyRS;T&M_jy3r&AVnc_-I&jTHosqVptrIco5VFSygU zaNl$UHpthAv;rs|#H+HS)a_&Q;U&>4D$@XhQUqMe=sM8tzJ*mZ>oVV-v89-Xk@z*f@^9Jo~ zj+g&F6Rb`-zJY4?{7qY29W*%bOtYoxrQYiA9Z)rM!xb&klctNu45MYqmkH0W`+*M z&lSJtbbAUuvY#LK5|@{POHyHrn-(pT%#+ca^NFZ+R;gk^O4Ho zI>@}|($qQNBQ_oB?7#96`4uWB!AFkk(uw>&q9gNqt1IP#=4CTIhEgeB_NOo!Uqrd07>i1wPW+n$BYT$VvYNxaYtA zHh&5%OyzNOeLX%Rowg2xW{fcPfL5hUrh_E)5!Uc-+zJ^4KJwI_PHosnic%C%zXrES zcLN`>oI~eHZ*uAO{~TiV}fDaJ(#}>wQB6=@*-t10e68(dFDAlfCaC$hY4*82}>pj?Ul(i48@N zK87a;0*L3T%V$I&AslaaC?27OAgSj~DgZ=w6J5>?VmLAiK?cixRR$2V?Q{k&hXA@;^a;}@DXiCI*7+d)@iiE37*qFG8=p(bUqy- zv5z>KEW~qW&8oPo;3IE6>DY#RWWNLBJdLv0unl}fWg(qbpO37cFbr`oTHYQGK2jS< zCpFGTEH{is=AAY6tHDQhuc5R5%14Uc_xA)JSs6+v^83iv?u`DW>&lhFDmLCm=ZG$T zmXFNW9gCein)|K-4%tz==nP4HWK!)|?Ai0--XY*4h5P8#M*7IPxF9|7ks%3m7TZT& zYaU09$Xflp2o`48NxHrsAL-L78-toyPaFuX>YYXhN$Ml4;cb3XG8%lO>O7s=u#bGL z(L*1JlzOund}QZkI!_`WiQSZo*T|fJNqB#x|4llQcY{<)!40zec!N9OBkk|fDUI`y z%e}3$;Lwo2`L#y0*H&@7aU(l{arzaEpM}SG!acg2`_R}Onu&)-$~s>?I5bKh(HXow z3C_Tt^fFT%3m~eoI&IDV+#h+)@rs0ZcT{m=n$>)fgAL$qJYCKWk{BX|8OFD7zXK5e zlXM0zh;}Z5ByTt!1t3B5y$nSlMI66r*OSv{3?jv48-VQfrpviOR((yt8JXWY5s&d5 zOXv(jkwOC{?91V} z(~H1IdKA(j68p%6g^_s9EJ!h120mi@n2v48N9w^^t*z9@w+t8#ZTvkBn5LQyb|cpYJra z2OsICN@uZsWZ1fws1brkvTtBvDrnO6_4r7JffTOr&2lH*pjE@V&_R;=2y1xSbE}oW zM~3RtsSW!`YTO9)k;5w=9|a#V>`UiK4~LAng0BY8K-s~@m^GsbVd z2tG2)oK9(+kLdNTY6^!&)6flmqP;eY;}`njR~Vb)Z6hxWST|`AbUF8-AmYMhURB`ii}sfq3J`H_I1boO8Q$USM9&EO+nWsXso3cruM zoy6$hq+zMyu!<9ebdKoaXZc9+G7IcnL9ZkD{7B1IbcUopBJf#(JsbM2uN?SDg({ue zNFUktZGRW=ky~1H7TZUr1}Ng5-?1ty6Bgzp9lE|AA6a3ki9r?@ho1o-`PPFDlGI07 z!?RTIz5qTl!;nsG*heD5oY6=8uYZ>VACWSq^Ca>S?N*F$-+nH?b`Ca3?V)re?*=(v z4L8Vkod>#ukCa){DUI`y=F#7pz@gFhP|#b^UK`Hwi{4FzjAQ)#=6EAmHx9??a_&Rp zkkLLoG&0Rg+Cfhe&d?dWJu%n7p4j`AWCO_N+CFka{@fo~&G8e=KGKY1yz2VM=Kyk3 zrY}{_4dQxbJA(L|DVPCB69qbh7euKGf}F`-@f1Lw$QO4Kfz)vP^6%_ z!L|~7B*Bpm;_;DwO@AZP_2(7_;3KQ%(;*W3$SNns&)ROB*}MgO61YDyb~T;-S3a`+ zw$fPek#ig9M1CLfpUddqj_!}#;o`rwh0YOO{45_aS!Iqp!7{J&AK{Sg7fEMG>LVUo z7(LrI`8p)JFQqo!2Yf!AJBC(^+gE*?UMIHR8IKPBQq2!g0F39v{)1J{W@r zJ8F!BR#_#}L6Z6iYj_KVrH{ZzcBj*+4f{x%=?e4_BYF3E;3I`ube=>$l68*pjit@e zew$%~?0c1t*ilIx}5va==6FkZm<)l<)lJSUTV`Bygdmv!Jho=-u5JbeEaGaC<3|5@x!x` zQxylO%npZE0?2fEI)fKvf(3$HZ#A(qfLwlN8zlm{>dgMV zQ^h<1f)r*v!<}qg1zpY!qIUQY&WO#tr_KQKp_tu=H8FG$rS1UdG-FdDW!znQjqB9K*X z_4&w~M23%Cw!E|eeB{GiI*7+d6s;JK#GAZ*@C1CMmj@jpv5&Ou#(24m@sOXNz(>aW z)3FWtNcI-Sd8&MF^nCD<%B6H#eLnJZ?*P0EyyRMjw&=y>S-! z$nEuX_FwtP8tM8a<0Q;Gtn;{GucDq~&KigSRJHcG#0B{Rj2{l5*hI zClN@THyb2y7vodsQJ;Q80pxumUCs^i^w>rOQT4Z53LrAcbOtX-_-q7u*DCQEfb7hU zQ8fQ^qbJgz4YF0Yk}(ML*Lwj(<|bXv4N{aCgBfGrSoi`+lY4XqFNlLHf+!?B{{$dY zdXLfA)pSHZ0BJsy&fo>pY&wX?N94vnM<#!T$c^A5VYzgO#6GgspYgXe za%){TfsZ`7O~*FmBd7N<&Ql%TnG?WA3XAEq`h3J=W`Dd4UO#>L6nrG0j81Btj~uiO z#k2M9Zw~?Z$jw)D_TTwPaDoB&NN^>c$nPVr?u`D)hP?NIRXqDMog=#VSw5omn{j`n zb3jlc9I^v`&>52Y$cw(~v1eCLSm}U|bd^b<7DFR_ByV8zhu|aIo6}irACVQ#L5*1W zebE_MnCZ%NeLX&MX7wTry0vSSHMFW%J32^GA7KrzX5`9W;3F5c=+uUNQ*5D&1Ms!N!d}N06qgpsL zzIO5*C)#U+1KEd$%g%4OIZk-ET?f|9IDNXD`_MSOco$}zSaNSW^rWIMox$4^sU_GG zDaV!maBaWa+sjo1QXMSKCRr`0!clBo?Q|VJyH1WL>bX(cZ(taS{;{^>2S|zky(+(M zD^N%xLU>uqgOTNs%`t0WIg~21R>V@inuF!#i9%$VHoXx4Kw{@CqMn;2<2mEIj%WWm zy8w&HRU(9!rC>F($ZFeN2bO75uI&@Cl!S1wSoD2`EQ6*TItuf0V+K*r&GI>Z8;0cX z6ubnMcg{ozFH2eovfRsjBn_wRuNE>FL@X5>rRy`3kxiE4FKxaAEIJO?ym?z9iAPOj z%Wff-_h^UBBkC@NPaPIV2=h|>JDasuM3871unc9w1I8Vbt{)OVgUaq3 zON2XLoY4$M zw@@7jEuod+riM2M~c0iHh$1-MAJs&Nk159kU>i$mHEIi#9TL%;?-p zV6UF5iI~Qz%Jcokm2i^u?w*n@+IdzaFVp_L)3GhBuDo>#4UI6 zdgD^)OVU6hgtsrTq1czC%n&zVQJOKdOvI8M&cUMT`xu9DX-`Ld@!U6OqMn=Od~z&? zn5PGu1BbY5( zSvv&JPGf$LJOjybl)X=vPEQJ+{)%b{8h8z0h6_?b`H?NRrf6 zSOd&;GRF?HR4z?0hO9V<}D{8)s7t*@*GkOR+ra}`(daL8z zF$x6lKgN#Vl3KyV`Df@5^Y_EXv}H5Y%MIIe*%bl z?vq5h$$#(7Yc3QB<$RqS-5h;<_eT%L&%KWVi4fktn8#pW9(T(L2A2F4E2fHAB4RmM zdh~pP!zh{V{0mqnY#{2nSz`1VwlY3!^k$gjz^z0GFNibIyt^4QBTFGuE23lp*Y zILyIvYCGe*tNSaelwn>hyoq{lmhth&u_040O^gPXJ&TDDUY44}$dZ_*lK~syBa4$K zL@aj_>$R1C)fl!i_ftqOu$5k8i6kCd(HqZr>(4MpnNwgZI+KYMiEZV!u@LY4Xy>gq z1zS06N8~nSD<9(+Cv9&z^B}MlD@P)(K3f^l!x+E$nVjSr2)0rWWO_+Qyd53ih^id(Uf}AuJ)+&qO`?0xGs4Oq&*mdOnJb8qEFDhEsD^83iLw)NaZVDXlILf3P%oLzki zLx!fDIRo>uLO_J@vP7mL%ZAS`_zhjstD#yVmaGg87K?Qca2WFw1{?&ICRIc|H%nA3 zO;bjTVK$c4eBj&=UoB4seM!r$ayl&1*Xcw+J?8b< zBMp;Elw&tQyVNfdNs^iuYk<$C*PI6P+LlAaHf&yV4;G?%{d?@GE|}NLn?#^Q<|V(U z0Oxeny@7$SW9r@`GI@8*G{%lOP??x z)}3)GuDM&5088lUV4|M;B=NFhyl3N4M3MsZ#mnZPmWocWYo}1-lL=uMds8kpZEYX@o2rtXP90p6sD!dn|xv_tQ zh$TFigT;Sr77nBTmk&>Y<<>T$o|{E&Iin#xpJ#6cmi^I02rr9vF0wS+;%^HZ;ps}> zSP{#c|5)m=mGZF+TdCQ$1z&cj^_58Cv6YP%7_aa>YuPReY~{yKB1K|bnR%xJ+REV@ z&CS489?SaCoDJE^5~i(~{5)X-#|F3mw(NmMy01MTu0NhC>XE35&AZ7lEtTj@1{h;7(bJ{!D5TUplWbPU+a zS{ouzB3n`0P=;%v*MbO3*fCGcBr1c@^I8Lo`*R|MmnG`~ zvY2?K-+_(r%E3!&B9;{;_1el|4~DHwsXF}{?yYP+P9*WzivL>1m#7M|Z90LioK7K9 zB(@d17P@FF9WxzGz*fd)61ff8N=K%x3>kW81K3K_OGI3KwxaWd@f1m{qH<@jl?~U3 zsK(jKm%a>JiT3@L4z`k2K!pF5t!T|Z)dOs0dm$0YZ!20%TZ!D>eiJN%7LSP_(K2A! z%I&9&zqa&xlC~=xzvrG4A(Gn4#U~6~*&8x;9o$>-dP~GM(pDC=>}vzIa{Ln!#m z?yN>zIWntYE!fJW7DS*#wi5p41KLWHV_H{X#|%{=GI@8*CMCFIj=UsyAMUN_wI^a4 zXDfOGzu^0l&E9(Gi+10;k2y{f!3@R;AMo{tEi9ql)kHn_NuuR;0gqOPN2RvV7tJ3; z2yb8FA7fwkRQ|wI(Pm+Zm5Al)Qw|oZoa;D@;J6QBTI7@vDvq&;piSLZY6VMP}*0$kMKO-Cke` zQYJ!pSr)xSmV!&t7l9>m!RbH|OVS$-max-|Yix$GMPFb^^CjxJS(fi*C{M<4)hEEB zvXltnWpQ|eETc6;BEeSVlDlpZv23oW*H+pFGHm7kp zB<=MATbWoyL^aM zmVwGUB1p6hShjM+XcB%C)bWpSR_@div5mBq9N|qT zuoWd49eRziZKZk2U382^bEWNJk!}$X0rl8Q(UupOq!#jMD6}hAkw}u%R#*dccZ)j+ zwz94*5!EDMpwKU*;Qp2{b%6`c-5phUJ(bx#_%fO**&!(hiu?LuVo?wFbtxMOZB z8t4JGa!8+uX`HRZw|k)gCrR$!1+k*tH?ET7B&n5RobY+M1=X;G#>5l#+$V{2CF4P! zO2v8j?aBBgB80auBPy{k=YP810hYcCZ)b>DB0qAlv^viC_`QD8{a|336hzc>vvl5b z6;Jryi3(LP#|PIDA-pWTKO)OarH!S)vOc<3p@=2)3kS=2OU4P`wT)|QVCfb|)N`}s zYurSZ=gzU~fyLw~5yH!&_61q)1V@(x%i7KBK8jd^YdKifL@`*b?=1EQmOhb0JvWQh z_+n)7X>xcru#Al*LU>uEYLO+*wW|kggdPhP3x@x>Pqg<(y|&VQIm1@6e>l#Ddn?I) zL=um!th&Afu|ldB^aoq{wUkJa*jB!6>y2C0%cMmmU@Lc561ff8%BM)iNxNiT@CC4y zG3$xA`fQ~~;z&GicRH=V0Jf66nTTqft(;iIxVQ3P{{~~Q6^)%l_+QzIPnBZ5TBfMqN30aLJhnbp00;P`!&K!iwYD_i^+&#jEI$o~bl zVsMIxZKSQ7OgWGPwxXI>2X9;mxP$7)#4sFT)~@$sz*kv6T@q)tK~c;gyxpu4y?$ zlBBl68lb}!6Hl;}?3+Yv!?rT@ygb^N0b znY=ru*$>bY62bT35~oA>^Pz%nU{2;pauvW6#y6)(l$@0-fbT^uc9DOtn8l4*Pz zhjDbU5xzTS-?fm!+f$vS{VZOM-c6f49{<5lg{(4i=dS z`*9c>9d3BSHMX#XsOM%;IhTSV1B#d9eIonkLX`xN( zdu}B%oM9_d!hVK;txUd5B=Oivo2bo*bt6ZAIoOKY4I)KiTlvx65N+lBxxF{QR-W7@ zavQRh0TB#aF`T&r@2$8N6LIy~if0qX-y05nZLkzsV`E|o+izpc2=Ve~Lfe!x^%2BFnNkZ2jOY(?+MG_)0ispG=o_;vkG zgh*;D^2LnbH(fg+{|s!!y~!bFAk@}1mhz-11VFpX6i9S)!eUdEwhw+ewvc@P6=u359B80au*^IutHNKJ$EDzh< zi4&hBTR2$sr5X2D4lByL1B<;TQP0iN^~(!9J`^T@xCksZIz$LBOPV~+vD4&lzkuby z^-)`$p{MI7#9M#o=z5m6Em!PLi+3iF)pnq#%oNDjwK4S04Iu`V0}m+m{GNU)n6! znh2-j{z&I`mVfTjhwS5EDeTL5VzDqTJPTNa`-pmOma>$m*pOB`uUiAlt9T-WmnB3A z=eUP3!5dhfo_t~?V)+rr!P4RdImNVywdTy4PQjC`zojUj#&Fk%DB7~R4 zO9ffF_D{v2}BZ)tz_i};N_-t;Phayl{Pj+ zio~{J=VytwQm8a<1K5hpOd_`-Tj{)sanc6740;B(@@@_hSD&o}^q-99t;%*Q9k3P0 z1w>TiY~_OkWO#3R`&W3;eTZ-Zg%#4z*e*u6OsJ3a@3B|L;p(?r@%4@4kCg? z%YbDoH(VUCdx1Yz&xGSwYb_BXsjWQMW87PL)yCQpY$YX>h;5{;6eL7W23xV-Muf3# zWk6^%bPS^&ODe!t=ItT^>amrBB}$kyZuTP$XxFJ&B1uwPVGr<}TM^jGjKf51!?yBm zh$h;KP1LJHU@OVTh(L*KCH{L?+yX)mu3P}Szf1~|$-84ZwZ$Ft+szDocT7kI5z{zZ zx$$Zlo{D9kcZql3=l|`#taryWlRJpJ<;V&_6)Yk5Uqn6kNn)7t5kqoMfJO z!P}QPjJ~{W_4x{%@NN-Rn?xr`Q4$AB*g(eph)!J(M*vHY-9$Y%%flXwQ*q5Jm3zSA zdXNa=WtpaibG)Lg6fN?|gy}~_EH{!lSYo_)V@o?$>{SDnG&`c6n3vKVP1i?GMbXRxWS^-6jpVyXF$MZBqVzB{JuHp9Gh+a;!fc_}O=l6cH3 zdblh0rs~_yd@!%6!9cY{>6)68ElRxf<()JWnTAN&B5-;f6$!>Dlz{A5hAI1OVPS<*-N}W)lJRm{*je8YV5<+T{(j>(+H5NmBD- z4RBxFsas%P>u(XU4V%{~&#q`*AKc{Uu+^6r?6 z7&~UuT7^z?XI~gGSX=R8XvAKiAU6 z($2om;txH#{;myPzrI*Of5h+Fc8c1H3iItOtt^%-wX^*%pt7=Z00bO3E7z#|?|^!T zH76_NE&Uy!qu5Td?tGi2b1ZEv>>aF^{TK1;e};(4jcz@9{f;=ea_tm_L8C@Yi~Rrb ztJLVwQBha1zQeC`(#3M_Tt_?0xhwz6(Z1cZ59DMW&ggCMJ7-pRHFJgJiF{6b#kPuy zk{t4Uq|WMx=i}yiCOVBM=>PjTW{)g=tsrN`XXq>HDK=VR zX|vGE(qZ9JA*yo2&)$GK8eg=d_wTSRb$XU6WH08!Iw^KgR3^emS*;zE?D4pwEC}+m z6Ah}8*YM>ylR>|aNaje&F@;*I`5*(uZi;He5iN1FwXk-!v|VOn&Cfv@x4y6PYAcwDr#A61++^lr$a54vpE^gr z_LoZ`qK6M7qRO+=cJI*8zP~ZewIazd6+MkTvkgR{(os&{r{r+H5LEM z`MkAq2;}T7a!)h~>u6)?U~Oe#?>NVHp}qb8^DG<| z+KM0C_}Y_u4E!BB&U%HVLTVfz+D%bkQI!n+|G(QnoT>dIJjOuh_?3fy$F2Oh5UwS@|e=w5iY&xn261Qe5p^3~N;EVs#S z3JTNFe1y0n1Qiz__52J9)mG;v3OS*|cMzqBZqWV(C`z>XYPLeq#i?kjxr3sh6CWY2 zNJ2%{&?fGnC{wy3QPdGCerET?u~?PsUw(oP$_F0>#kUW9HCrKAqsAOEumVNT?|g)~LIsbbL`xUfsCrimB??8M z;_Y5TtPXiO{3j?%GWlw@LNFFPTsN)+McM;CLR_JV$1!^Ki$WNS6A`5nMO~qycZC5~ zUv!8Z1d1sq_-eM|J3XV~b_5Or#h|l%gt$TWxU(HqsR@x_H zw&;Ok)M`FLTrnGuBR8>*IVfIDX(dr83l*vpx?;7f@r*j4IA+0DvlW6F)qZ>SU{GYu z=Oe@w%kVh5KijPjiqK&?5`~ITF~F6&s-_t}X%33jW_)!u6%E#O$2DsIWIjS%;f9Ky zjYB zV_Z68wd=KQk)T-fk*{Ve1XuU{U75Q;aq2rCA+89-(I1gpfJkgtJw;{)m{6}C%o&me#u9OD?;!%X5<<iBK`um{REUf3^`6rU`s?H5KyF zc@>}-a+!}1SHz&g)puz$DBg!VNEB*9MJE|utlspv;4~<%M)B2bh2ZL*k$UC}C@#kH z5#ow7cpOiL`{PPGZL_OH(Nw5VzT6(GjV#9E8a36Iudb%z-M6O=L9ru}%*p+;BH5FMaVupbtU>zSJuE@mWn7Ud4d)mZBAreIkp`yY= z8>{KS0r6 znU4@xl;d%na?82^3X_uilH<@6DtdaVV|9P+wYcwY^ntHtD+JetpGK!gpt$vwj}TYL z;FnRm+|TU?#g2PVC5qNU#fy+us8HY0JPpQT#1p=nt@uihMY6q1I4CL$`3P}^EGj-M z8aN6RL$17*DB1`W@()|0Vy3LhW>7fXVSx^g9WM-A(=TYC{_-eM|3q2NY88Nw_$i2Wvh%4%$!qj}mH6=n2T+-ZA#1Sov{`3P}EV^n+#v788FF~F&*MA1&D2QgvlW7| zm|pZX4aQ=Y6(1q4Xo-sGJ;n7wVcS2V>-PmknjRk^ zuFyus@te7aK+(C*P>G_0P%$n>6&3ORlY(F@&MWiPY{f@fVWCsk5EMt$_y}=@4k|SI z)x=#=cDb2E(NU;)V%ZoK5i48121SC*nE$G)sqpuXE(1l7JRc#h=!A;7osuWPShRUM zO>!KagbJVHMyQaxxd`_c-;4O_YATBRxZp17bSWPpuFyw?^~HcApqQ0yB~f%1D!gnc zh0=AWn*U z{krzp)9wxBBg7R(sJOPa#%@q7+3X`xbQLPzYAlb zxrdJsR}4i(J)2!cplD~iN22H^RJ6LHjAOCjMY9&5nBd4)vlXvsh0@kp-$8L{EgvDS z7=a3{oKd|&;XEr`qUbJEj2TQRB8=PPUN3AOU(Hq&(~7(dIY&^0Ea4->6{Arxx-|JD zD3%UCAyF6z6&oVz<5(m|cMgQH7&?ZpuBPI#*MU!bd^6B_plB*tH%~s^o ziqzox_#LZ4Pd-9iF&h=0#TUnbB2leKqUbGDn75!5#wD|qVJv!R^3`lb9X?3 zaY=`d5LZ~C!vE2Cd=mAT+((I`k5KX0p&rhtsGd7bKw+=USF;s5*DCPWAFJfQn1N!I z8XqC9Sb&P74c&5JEM8UolqmWN6$L9PMXmB>UqE3|r{ceAw&GU1_o#>;eZMCtE-Ld8 z;tCs7f`2SH&Xr|{na zh%1(%BC7u$?4#D_HIjt%7bslP@#F8Qq(ywi{0SFD||IuQMfMw71{Oy$uJh3Z}Ji13O7`|d3EMJD1w6rN)(J= z*~u0C|0itZuk5x~+N;1=3=QY2*@}YP6R6O&*1ZRc*-?CixWXM3-UAIPK~dsAN}?Dn z91E41ltQiXdHktgS}4<6~+FP zqCxaH{Ioe^HDAqEl&%Ry#iD)30$?m2tmh-d6&p~oMfUw(P*}~FEl~^=Dqe(9ip^b{ zKZ6++K9{d%D@LmCMa7rsEBL9}xy5{hxWWq+vbh?;px8HPu|#1kR3zF_3V(N_F`!Tz z#aFWxrs4jmD04Ux24hisJRc#h*o=x=vz7&bBCYcZiDH;g(LPijukO%tKPONq^x&)6 zid&fhs5sb1E)NvL2k;T%3U5?ccJjcdf>*U#Cs7O+Dtxa|iiv&WI)I{OJHDE&cx>v6 zid`OQ(?JoY$47`OwxOce?U@~6ER5@WOB5r7icSrw8MS)x{QIB?YrZj#6}=y)POR$(Q+Rwjv^L4JtxTJlO$?oj3RhaYZ01b`%75 z0!7)$6p6xAsJQZono&WceLO(nbDpneD`X~6ioTAow}2wy5+5P12tq~r+NFy@v25oJ ziNZ{%=&esF8k?Esfg*n&U(Ht3SJ;hXF=VvtQcy%3<|D)vdr^^-+!Q~j&D;Dyq8KAo ze9fa2U1tW{f?~EGU(Hq=U$YPu*CyVY1d1Jd_y}>uK2(f!d9xPAqF~W;iDIl!@$4<7 zXtiU)MNss$=d0NY=X=Xh;S~G4J17RP<|D)vA*krDbifW2t){${D8>mDu{Eiy`+L)I zc)uvK;H%k+%T1`Uu*_Gx28yrq`3P}EC@NmZrs9lxI{2$ZFL^PT6NQQt8A>tPV!(K~E|zrVtJw-; zv+1Zf+AaWhtP^|l5#ovnR7A<2#;3eGG;1zVOcE-l526&y3kN>{g?t;nnyuJ=VI?X) zJeq;OOM0WrM~EwqqGEx;#9=U_0_wJvC?*RP*X~l+#m#{Zk3mtpF<;GAT)1zIiXACC zHp7g1-hz)1S45&h#ljswJ!tp6i$pO+s5rKTQrwc;=?98~weED4z=RP0_xDP}rNeFBPQ#e6keVf2tv9DimP0*dLSe1y2- zBq~0XEO-ryn1Znq#dM(}+=^28UyFYPieYc~YPRAu-GMl1^uQ-58h+*@#1(O<`1~Y& z2q*^Mnjull5Gpz@q7+XK%9w#-&;!1jt+;w>8qTQ1qv;o5EVk$K5#ow?R4Df}!--<;)*k<82xGQ3mA*mk;^5D znL#?DgiW*@}TXO>itmcDud`6leVT2ysOsDqbCOO$9~xYF~+Bwoox{SZ(ZSGaYK- z^HGYPd^KB9k~s$zN;B=ofuhY;K0;iPgo<u6;#;w`neIt;(pIWiNZ>#IBP^Hj!nOhPZqT`;;Y#T)wW|$(d&DG z6DV?r^AX~TbX26Y>{=h*v6t%H(w5unePX=KE2PbRd|A*%(^X&Kb&(_2Nz2&|!SgJw zycVqM@%zNq#tR~YnFHvAo9`D3~|V+q3~VlnX;S^fOuvE`Eih3#_Mv1WdtBZ^Wl9@fb_p7 zEoVatXh=cR(6<2D_*j}D3UND+5RJWktpE}v|2Uk3WHR1wGD&weLdLK0+XIl14W#94 z$TJ!;`b^>#fCM&^W{5&$z9Gb0W5IlYT&qw&$3apX2`}|i9@7x=U{;e5fF#Lj@a1fX z;K{WYL)B#g5?WuHAqp{Si7y&eDxd2FkQ*9SH#o={#v23<-ye>Ukk08>0kTF%TF!=K z?8wJOYu$;7Cjb(!FU=5z&~Hp_6@PyZKy+LW=W~!q#!tR=%zGh3)^Yj{fXwugma`$g zM{*Dn=vn&=Kz8{^GejXt({V(azG%A>AnwcGeB>Y@jNj<$j4?z=f3JyW0WxKkw44oj zK9+jOCGNcESb(UxOEW|vGOG~Ms@E$$c)PC8#$6qc|9Xe<70s$%Vmz#*F1|8!qGR$v zxU0?HAq^6Fi}4ua_IO?@-df^@P-izYgnX0j)2nQaR@IA)&(6?7T*QxgrI2(iT0Dsqp1&M{!b#d~ec`RKZl|S5 z|MV*3w7dFh@Xp<^L}{YJ>tpz6NSSa5j9H8!Xl z{Z7?U`xKy{|6ZE<=T{nc&r!#tbh6s=dC2d)6P|5D6y|)9=2h#R#%?=``$KDWvm#3Y zmbx~T(n!&zPTw7uI=5NI&EO5!+vEcIrS9+EYMfbn)oxh+gH87K;#N(6#!t~Vm371! z-gJh~8JOYnlceSBMdSER$Ua0i^z-`z9R3i}Bmx zVK>zgl4*X*5g@Ae(sDK=JnINT3YDkL0?4S<(hO0E(O!h8eAsykAj4*^R7K#%~O# zcT+{kr$r5%0TNs$EoVcTeA$JNj+akN1IXA)X@)4o?L0zE8ra+i$gXsMPYzPR_)*w4 z@1_VzKB_(vATMr9%h`}wnL81pTDV~;Kn^^VW{5&GuOp-~z1tap_#M6-$Uz=3e#PV7 zfSOv)ZL=c)G9gA<&W6N3q#!4)&;0<%gEP_$QApB5gk)sowg$-0Z_1GzB!ls@i}nc( za70X>n|A|F2mn|ng{ZMRgKDBgW%9BhSWtLfw|Ct+fntdQn#6I;-I=fwZSjLvh~Dno|$ zT2=;mhWZ{&z%%seTl0(1eg9l9&G^giJ8OKKnb3V7+APfycHjH#sirx)ezRS0i8b?; z)>qMe8|5hAwc2IGp+V4nj}MdvRk!;V9G=URy}8hRx8E;K{d4ynEi(bT?}=%vl%V@o z3YX?ptNYeJu?d&DapB+Z<5G8A8Y#NeEvPvRdieh30T-bAo)RZb`Md7>>B?tHu>8w4 z`Yy+X$8k$KnL?I#d5Yp&! z#ZiFt+PAPX2iePb;5z3j1?jBwDGea+!=>eHh-DoIghU=|TpJ)Cqof(4kY$Y#GN4{^ zF+hf@h8b}XAI1~1nwu1GM6@@L2n9$xb!j;pa&P>6gvg(aE(S(svf0J(5YTF!<%8a@jln?lk80HT>C%@BpCXd`5G?9niQ zl(g2E&q2x=Pttwhn{K*w3n1OQNi#$t`5h5bGAR%aa9?p9fL!yEma`#`O{i}kTh^@Q0FXi-X@)2yraMA{6ji#w zh$KAUyOo3FGoDpZI6zG;gXDTc0W#pVw44oT|9%oqtrq)T6akX^QJNtNaqEka5h*>h z05W|}P6!9N%Xqk8lPfhMtyY%sPdtef zj@Wkxg(LQDdkfo$QG29G|Fn(h5<5Z%wh>05(nRrXMBFtkG=|(a@qvl`DMFgVO>Dt7 zqDQyqn9))F(|5?&5Y5UU&yergYj}p*SVXzOHX=4&n(>$02(49?D`6WkEX;5{y5rV_pyXnJu*hbX4BTfDD zHbQr(J91y$+N%%S2#p+RUbVIn3;OKArEd7GiA`Y5w#=7CiY|3Y)KZsbGC2{p5er^Q zQ~qunF=U>_SXlmz!k@q3R?T#4;o||X>}z3X;k0X{9L(^8(b97E@*f;QJ3rxszqux zzEBAu))mqWQApBsgyFZM z5US?`kk{X(8KMxMIS9!za@-6fqTcg)Hx3f$D1_v8pgv5}+uwgQK!zDf%h`~Hi>YhG zv8>VxY86dl}rR8i$+XOv? zEM8=i3y}AD(hO0E%qoO539qLHcjOdcZVFJVsxnzv@?rI^b<*Z6A$u8?H(jU=VAQIg*TFKpn5bu48jNd;7N>`CYr#^pln4>E80|$% z^O^%Xfu-f{g*`coe+a|ky1f*~l8WKU= z6c_aAwU=gKNW0S;@jAKLlo>*vC1w8{JWHzsqurrTkJKPS{<2R$TwmQ9`t;82h%jNF zUZ7=)-SyF*7v8|-cCIrKP(`2aVCs)a@!RCepijTqok*%~pDsAS>=g%-pike>hlu@i zpWb!oDeTh+`nqq2KE3H6BCvXWx+e7=SQ)qZ1+eyok03Hd*FJ-Txc0d!{Zu#yvrE>`R>M>OLov;DWiil&ELl6HX4H?gBhXIrjEQI@Rv(ci$b zb0rZXYKe(O%g5D(Enr*}I}`P6%T8ZqwCwND zr3P5a4Tum?%Yut&8Di8m5iB~Vdp74R5tkU2!CPOT<@kadZeY1`k*H@|x(;lImWDcN zhrqHYjR+C7jJ<>w*J-BC&;&)OtBvO@@BVL*G(mQj3zu}UoIBUi&T{U`!HV)V8p^bh z{b{R%s)hZo7{PAAL5D~Z*>jB8d>d1=(oD-B#j6WDg={pJ?5~OMpHS9KoC$l5vptBE zKkYf@@1|%P2lS`GMqx^SBCe`E$C~4}(6`}6{v3#UJEV#z5_wps0K4fGjn?YGo@15? z5%o`dj+GhbcEFyad>j!ezUSC-t|Ky=+&`5MQ~c^wB8Z#ff;~rp{bdaCI&m4VlTjAT z5b`Xo7(?yMT!LJDVb4)Cj|ln8J;$i2jN!wXb;Xg^{rKz1vP?imrW6sI{-2_M{-#bENJhV*YN=vFQ23mvB!={E{$- zTSxb%G42UAwWvLZ+v)M;aKQ|eNtD*J?+H6M1}gp{*M zi@_4|^^+rK@wv{hBpxWl>vrPc;pf5fq{e4yJ=^kgp9Wg8;%|Kh%UcB^MAV{r9WAn7 zTeJtu?P9}CoW060UwBZwk1_f9WBXAXYBz?$~PiJ)FOWiEz>oE zK7hsI&5FI8rTh-VvRszx2V)-`^aM-MXQG~M@mt>)Elo>ZRlsuUClMlQd4C5jPp!RL zfhA2hKAN+|G@U7ZbUuXvb?<}TGE#q>4W7*Zz4q0^58yN8hn3n z9V}0de@fvj`41Tun}Q;oW5@OS;GbR#h$rgVmf3Z+(UMwuaVl8GB@!W`mZXPhv8z1L z1}qIPbidD8?ml5y6c$oXAh&<|0so}msZ^q#ZCNp^C0Z&~t?)a}+ZjZNs3qbFTI$_# z*8t1YyLQE#CH)!060Mn!-Zr;i0hh^WQq8CsrP zdvO&S2banFyPy2Ee@ZQ=vQPIo+YVK6mm9r;4m-z!ND}GOhZWt&lw+;F{{JyF$$#AI z)k^2FQ^@8lZCWL^u#SD?mSUf7OQig%PhZ!h10GbuhCPR&xt{1i#8uU&YdBrQs6|e# zUqDp&>MEj0b z>>z@;DK6;KW0aFHq@Q;X-tS`rnIYs^a!<;~sUTn0$piXyr~O38U-s$WTX&ldefp0u zB23t)m&a|x?t1x6;~H?e9X?J3RMDqf45i-BFudca-q5F~#}Y}^?b8JZ_+)w4HPEMD zKS#v=xleyk_73~>>orYhLZ5D#OaxY|Pghw(y_U^Y=A1j+lakYkOwqN^s{q%&5q(a* zgFgMjEh6Uc`t*1j9Yeb7+BVS@gB=rVoDg6npVYl-Ij)HErk1w`Iq+c$Qnaq7@LBuI)2ZiFOz*m?gT-J!QO~xN z`1QecZ0Pc%Pr%aiFcBhdkr@szl8DrApbm}08r_n*vA_EC-JUh8`aJzd4E3PPiMKY% z&|%N)P9%x+>02!C;94T@eXlO0+~~tjA)B)%1vT*V^Z=`accD+OGnh#EQ=h(Do%(cm z=;%pZpt+ttl8CFSPe0&%1$|>We2#!9>#}Plr`gGkHMASd^=^sD3 zJb*ska4r!k-luzhrTX-CkD}6HiqBa@1aVVb(5LraaURc=LF!xl{KwaU8A6_=g456N zEY)gc9}Io^z12j>U-s!fA3X0upYG;PgbDlffGeTcr+0ePcn@4|T{aT|RrKlSO$#x} z#(TsK=+n)8h@|TF>4F2CRd+@_^y$ZT5wU;n(}OP6#^co6lC%Z-^jZ6ez-smBK1v^P z?HhclC$4>u!iY@KwQqM#T>ECol;uO8E_0lS`MW-Sfr7~uxF>w>5>=mDM|1zbj<#0R zP?WVM(3LTrje|hJOSlzL&$jew)CnzmtG?o1ti25pB5KK$#WCKq z-(@IR2J~-k$XQaiGAt{u<=`5mKH3MLgDx{B>e-eVcd4KF)g5fy9>yibln4>Eq{^Yi zVaP;9uvisN9l=@7>|j`moT=|@9?nZOf^m89j;Lo_cFP#xxR@_}st1;^FGPr_<%~R9 zn!4{>3zm1jJ7;i~$N+}rJ3Ysox*o=9LwKJ)Uz$O-WcK>E1$2L43@-0M2M)xzdl+X z`8>jBtLB7E4B#wY2N)K=eCipVu&hQnF6u{!dbZ`%1nRdbZzpvP083#E5h7~wQb9}Q zy2`OIE+72;A~?&BLkvrVViwM^quIUiT{N?TiF&rBL2c^QO?J^aILEq&5h0?MH4V|S zd+n`$&;(6YAM%p3ybY^z&!JmHeSkjTW_<(LEvV@bNg{iWkvhDZFc$V4lT3)Hf7){tb~o~bJ%{ReB2s+M@mjG9G8=x|JPVc{->F0p zH^l{ejyDQtF{Gf0HGXCiWWfv}&(gULFL2LMlxU65@XVV>g#6{6qxq?Feb{r?WP4pnxbL?A9Bvp6MAvnMlz7<1Y&*9`w z#Qu5Dkv^s|ii>Vm%z`~f!%akBwe}pHvTEYQ7+o>*I;`VYeTYoaweKyp_JwX;U<`W> z=Uqh1-|acNxy-!>_XNY+!^*gI^nC>5p3v|c_4(kFEAKtvf=SFF>e=^%EmNrHD+b>& zTL9!}MwJAxoSR34h+3?gphaoB+7z(t(>mXhvpk4lSbFrMzOy-aP<{zm=5!+J*_PX5 z$72t*O7H7Qu+%mnLPRZ7o1$fNnYA@o!oGg(!dWun7?!8+sb{M?D(T=ekuPg}lh(5> z-S1N4vMjga9$5M-5+S0N(aq74sDDrwEPbZ*9>Q6Y&M+)nt*JS-z^@$VSh@vK&$c8j zp!&g~nSFbK#b-VdB5E;eiI!C-r}YL);^I}4IZNCHhUH2w^_`8s*}LX2$5uKJ^=!+@ z_G595r9L>(11v|^5Fw%#19h~-*yzj$OUl9X3pqW|h4Hg~%i(@2F&$i6` zx)?3tJ**n!@?Sw0>5^bT8ysDJ9ygZ*p!L!bU^2N5aW zr>ozf`gFq^y>S_R8%P9kQ(VxeH&=+qbEUR5WiDLv-usy$7cRGoI3l2mK3(RAGWO|P-Y#DbeY(MU zBB{E4y5IoU^}E;>`t*(|MC_mY^m!gSDDIQ%`~mv(9#@ILYW3+C=G4Q9QSxXXj6C*&-o?r&;q6%N8& z-`tw0XWtW+)*OcSw@_W9w{Tt@bcqnr^P-}S=Oy8MvvqKPYhS+c6KC1_YT<6rUJxBR1hDG%| z^||fidEWE!(wcn ziQ{6H)wU;Cx-=o`*_M{G=b$C%+KU!onXOKQh+1CjqQx;Q@-tX|CK~H;mi%mn<)S;) z+p5~`&;^TQ8d1--ly0TI8|XT5RA;cvxJ87BTJk%h#ka;S1+b)LZy3Z`?ml8zl0Q@L z{5bYyD}Dzal26pLEt#g2B~9HOe?xwzhzJq2-0h5(xhK!3gJsF+w27Q0J&$4WGN&x( zmQ{X$agm=y)Uz$?Y$oFzi~X|yK8(u+b0S34lHLU^tG-M!0?QFS`30Qi!gGdY`Z4PB z^vhq>CxJz^CsEI~?5RHiEmP-fePXhXn%GJ*_@@#m%~0iveApW(5IX2AyWR- zr^nr*`t%;3>NkSs+WmQg^dwc)r#p?jh`#c%EejxOLP-@-Br@Rx)q(b1=sf`X^z)@e z)Ias<=lXs0hCbc2oQM?f(`AqABU8b$qYF&&INuKX^w4@l$Y1v9b_RvX(5KI7M1%?Z^x5B@VAtv|*DeqG^vlhO zfGYa*_l?xCPjBfu?j!W+m6}9Sb^CO|0nXZ@sserb!jiH=}A#Uh^Xa|A&#;3 zW0`Uo7a5tU&p1oqJBFpTKh@h#T6bh5SoX;i^=wPI6Se11JZ_8Mb1YUNLPRZrJ<;Nr zD?bq|$1nMR<}BVH8I~_)SMjoO?LG>7TdNGBo^9FYHyg)g!K{1u`C#x}B1F{U-5V{l zeLRnXrGw9H`S@S^C$}#QOD8>Qj_s;Uod)9)8%Wf%Ei3C%e*<;IG3_02z3dJlLPRZY zebI6%xzZCXJs&H#e&|Oe$*V>k}HS5Rc^YL2obd` z7>JgQ*2ULgT=Z&fAIw>D-D?Tg zA)*%ZL1-~kza0&hrZ+PtahA*t3`_B7>NlP@qvCwP5`3SiXIpf9J<*cpA&0;1ANPa^ z5w(mRf)>R=^DaRXv~rr#PR{abW38%wHxO-2Jt|)N`qvQHEf~%vl0^0#N17+%IKK=@ z@q(11MeG!^(KJv~z&(fWuJ89@&k?blNcq#AW6N=h=I7|J1vUzf&O}^Qdyehn6VT_p zuAw1BwQ#K>ibS5$dV}5cV6P?1Vb5W=fr$F2J;!Oojux=zn7WmS6yI~KtDyEAF+Jop zVTyP5CxW;sF4%KCEI)zgDn4yiEeLrN#0(+NQo$Q)XI5~vW&!LujvOFD{&LR|sFi6C zdyXp+M3`{T(Y@|R?7%hUU**8%78FASRI%q!zDoV!#5dcFa$(OA8&4!vch4a>z^YX*A!7b+&ymwVe8i(K3oY}^(e7sBTpS~U3*d8p$Vu^aT z<$!@3S`@q44F${5^F)ZKrPpw@L}fRP1&g^u`Blym>C3R(aiyMve&1lu6|gw0CF{6$_Ji4KM2M(G(*!MkgVra3W&Oc5?>UQq5W{l!CbbvTb)HcV#>F<0sApRu z%U7Z0VET`hFfMcBh!9arV^g$z-1?y_Smt;n*E;=c|KzogVbOU_eV)GCa2tL%kl;RzJ;T~veKUj5w*yVK}$%{PyC)EskN*cXZdk}VR5)m&9M$%JNLl2bm~CV zvn@yNY(vZ4=QT9IlGu$15w-jnix$P=RjpxMT>k$D#5v3RFoq>ihuU-Kw<*Q%IgXtm z>e-g*YU|PRWmd&Tuy~y%LPRa^$D`#`L||`d9ISj})^V1Eh${Q^wXW261GVN19snKo zr9DKFNS|JsLVY*Ts!qr}NU430okBKe{p_gk2JZHkI|hAv@)08CPkp+cnjSWcn|J%* zv-bOAh`6fy^mWP?FzVb?yqn+6ZYg$ z3H0f69ugsc*{2^cd{_c~`iQ4Qn6OV@vaJ?=qCa}Wuvc)o#T5|&RrKkD*7w0A&1AJn z(5KgZMPA9c52Q7cJ4Xum40``(~Aoh=D$RR9zzG@A~wFSN%=lp5Plb%8Oe^3yv`E2`(8a zNb1#jM+waJL8pm&_B|oy4Al>|=oGe2oXIm1=M+Y(m@TMQFR!yW)Ek1aGYWB z)}}s#mLG49eaE!XL_OPLXF>JpyEQCN!2NCNL?T4g@?bKKaf1P7B`_{Olg~zRmW&e& zi)$z9eG`R`jqx*R^J_#s+cM$eQM^~1x6=s*OWiv}h^QrFDq0#nG1mvnCFAmB&XRPB zVQH;ReFiPJ>(XABV|HdlJ=@}I8-SMRp{Dp=d-=&kh^QrLI$9F91s?{>jT3$DahABV z49mqR>RrkSBPPUv#qAtX&$f6p@BFq3>OaI7>tV z!;-H`?K#GG_r&|$r*fj6ZRvGsJ6e{=I<5hWpNy;YAVe(@v(S>4eh%MV6KtLKowMvs zW>{8_rsmkIS2OW%T8vmi)Uz$|RzYZaG}#J&LtbwM5h7~YI~y${4aeiSOz^F%eCF5w z$>%b|qM=RgIn)=ICczvt+e6f|E%mIZ&(k#n!f}t@;nK-6;IDxyf_ ziW=0OLt*Os8?fi78AL?=)1Jd4xEX#o5Eep2itjl*)u^tp>#TB|;%yESLEID<>^Y29 zN8!0rJ2-AATqj?SGegL;)bk+qJkm({?0c~12#+H|{&LSz9Mw$?_8d;hCPSvRU)andk(ThG z+v1)h)L7*a>^Tq<}>^a;Eh?u|I zb4*nn@d@q;2j;7IaO-G!2IHPk*_V1ob!Mx6c`(;EEF)#X<7t!7}{$xLus(btc2IvM$wkD6X=_XA)9h5%p}##RJFi{-SK`2O^l zSBMZ%%j`vHIb6Tn4j32t8aWA^&vB@TVR76^eP^TSmWj{rO>rXX*%q_o!Dy+s zPIEq3>|KcvQH!<%TE^r%mq4H1xOT@;oF%Wg%09jCDLtHRFGr5oh7S8%eIiMuPwyB) zy?do+Er(N(lA_8^A)B)`PIa+Q?^8ZM3HtN~t%#IA_2~s)DVmM5>1SxJ_h}JvRrTp> z#!?K4pZfG3oucc&cLP&<5s~73`bmZE zI2lZ)WITk+b@MrXs40;=fKOBAPIQfj%f2K4EVEs3P+_UVEH z)JS;Q9Qt(A1w`zh`}F!|tFcd4t?@Jh`t;^YiNI?0>2ZyXabg&Zjj)8Zug3}^Q*`ai zqt?D&^H(l|KE35yBIfVpPG1U)F-ZNt`sQ(*CH77@=`aw{2@qgSc#IUE&l9|lWKYocDn z@-_l3)6FOC2g_4kB1F`Z>x!0DOE=)(YrB5`)m6@txn5TItbHq=RJ`^ZKYE3K)8bAZ zQO~xh+n+>B?AN;Wz~cUr2obduxuIoYd_gLV%j^MN9&?t|jSS1;D{gos+6twT%w=QCPE;}G=d#&JK`XZeR`9%6P=+?Z!PO0RsE??zub*_ ztBk^pB7bPEXVfL)s_N4N>YTxOTs|&wGeos-SVa_x>=XJ9`}F0md-9-9w{Av6{ZpSl zqvvNA=+h@?5|QG4dMA0RPp>&S3BSsF+Kvd~rnsO_cer~D&y|+ha|O6gf;%%q$g{M{ zo;pjfWfLAjpT5L^2>HuC-8{_C3qDW((uW8W_UXh|e^1Kj7a_$Kt}-NzEKf9}&GI&H*0y;Q;P2=wU*lZn7;_30j4 z%y41^+~^boYhUMCM5gH47wC>_pMT2yUeKqPS`jgS*Qa+-nT5~2eAFoF$E~AhwlMAq z)9BAS2Sga-o^-1YQO~|7Ow`_w_XPL6CuiWi?A0eiM9)hubzb&*r76SxZO#F$v79Ax zJHryxkGj7Fy|;9R>-O3aqMmKZDvd%*jfaybfu-w7B1F`3e?5-zsUFHdz_MZa+_{`3 z#E)S)+=IFwbxPP!36`fTiF&r>@>9yP`9WuV|6l$(B1F`Z>46rnZGEPKW!KU}D>#e) zE`~+(5!I((%zTEwT~cu->e-gAPxs-t1bJXD z$3BK7{{_|CPH(jvzq7e{1hTY)N;lPEnA+h_JMK9yq$50 zvlN6gENAF-tW@tL{{CswL!zE-nRcC;V?Uys#DQhvGa^LPa&i+|TrYKA1eVY&wQSDv z;3&hgVLkP1{J{~C_`;QOxkNqN(#`ESUi-HSGlV=#tBR>T$F?R%yWM#PAcHSles@QXk8|;94j^pQ-&VW5f%?CtMb@v>C0}N|7X+7*YHa;O@|Geke zoUtAE961IDC&8ZM>I)*UT6+$g9nFYh+!O4UQNJg;Vt=FyTrkJl5%uhQ!p0rc^XUdB>THMe zazKv=5j!t3;al*Y@NIwbEwHHe%NopCl1?%#D{`p6Ly~i9{z0&89ZJ-*Ehn`?@&5Kx zW2G@zhME!~qLvVEw1f>J zJ;g9CqwW$Rq89&cXql{CR0)=LPq(h&EPE3fmeL_qpB|eZi0?>gTtd{dErs18agIG* zv^fYYojwpDqLyvj(W0#H-V!Vmif?Y`EIug=OT&&-Ke$zAbOwxz_Xnb$ZRw~_?Kx^H z#5D%Xsc%GxsKv_%E%$q9^#@BE1LcF9#p4RYQqPcj?q%)sj}O4IwjWW?wtSfvjpI_; zX~21~^d3rth+5otprwuGgWF&UJ!2NnS<0_6EcP#{zZYd1JRF}B+nY?(vn}%3)aMOH zREzNQvHe$x5K+q-U$huqDgOkG!?dQ_UpUL1>s9vY&Yh`OuV}4q@(w!eT5X6Vkv`qT z;1u41a{^~Bfs_wA>=d#&JHLQ>hR1#RyIkng$LSL(f9lg8eWkt|@OK^g5c>47J&Cxg z`t+3Vl&@Rcj)NfTN&hOMNaTt5QtZ>$2IW{mpFZ7~i2A2Kee9daO6b!|O^8VGKD}=; z)u-?HV2yvqFJe3q#7%KQpKjac2+oDasl!s>n(s4>8A6_=xIXXkESZi{35Pzt*n$Z8 z%RW7!r$-;?(~r(0!i0VLm$Qc0U0+h%+5j%Mm$pPe6@B`EH*T0zNAs-`^yzU9L{fG8 zbio1oZyJ^eeY)XlBKFUH`k?Hc*ryL%x3UoW^rP#Dz-smBR@#<0F~&?Dcmvk)+M9?> z(Y3FbTKn=kmxn{2zGpiT^LKsv?&pCXa8F3q(5`vm*XJFtZ!qo&E2mI@yW`&QOkF^0|}<5w*sqB>xiKGJv&XNh^ruq?hy{T|*n z$n6eT%EuA)Y)hI!GFrB2-|Gn$-RVS#sAXy(T6&)UwgxP^*F79L%b^zxOYVE>d%^lQ z$77!!l||IEE$YLlXRDN&XZ8onh+HB>)G~fIT84KW7!Q`1t;t@TC9s%bxzm~Yyy0#_ z%2ybdt~-f(wq=|HwI(mSrHj9TNDC%HL@lF(&=OOT-5nZ-Bi({CILndORrcxkj!?~P zxJP?q=&-H&5lJF_x-v6`of{=FvZW>5<%P)7xd}do*u>!xpn*T9T+AK%nE0U3TLl zoEVPdj5J^!e}0I_6kGe;8t=iiuW^ma<DCoQJ=-FicOEUWW}_ouj6LP9N)JNRVzd{>`17!___=L% zip&?z^63-9GBe-e{GSv6@Nhg<8f@NkZ5h7|a*oT%HW|Q!5 zob{P**NU^`{$N-hB~kC44)y&$iV4LCvu!xy$%>^g>dI5K)Ux2wIkC zOkM_-!?MqYaF$dLhQ-C7dge{d=@otk{h}UG&$gtcQQto)9g4>9(tb83LPRZX4xnXf ztDpGy2D*1`F@>|7@nTq3ET-m|pUbZPFfJc^6ZLG%jj`1CPm4X@Y+??+m{Y0@Ij66wvb^tyB&*W@F6axK6j>r2$LEp5h7-*cSz-DnHf%lsfBMAXvc zAX-90Klgwp$g7b{IA>YEwRTm{@Hl;>Uia84J9i-L7G}31l0^0#hs&by+$a?tzX&N~ zwAm?Sqd7W28K2=vKfS{a_8gx(5GjA!bJ(a;dk*DhpTEJLqjonUuBtu9vaDG2ZF63+ z7NQRIsv?R+>g!P74fM>~rvQ77T1G_FKkYdNEsg67dyb>Sh)D50N575Ko+D%H4qSS& zO^G0GiVOA}@q5DYTO%gz>*C9qz0Bv{rrBI?k%QMmh!_mE>;@XabIiHD4+>vDLBZmL|eq*au(?F>^fM6 zwmReraZ^*q4du;+sS8XCh)bb$$Ejq!A@g4oaXK(0omIsF!7S9xh2obdu9!1O5>0`>EaX2+$s4HiAe7wp&{iYuEip0}D z&)7qUZ8MWd66w=h+eYI!$F%5p8dB1%*ePUlHo0vB?9;PG+kApP{mNn@+vF)mc?V6p37*MSbe6S$v@Z^y$0T5mEotr+@rV zYZdhAfnG$Uc%QCvlj_sE|9Ev4ruf+HL=ZQ{1%3Jr4XRJ?V)|(dgmm4>3?a`_VqNMi zrLDVP27S6lFcI>XeY)zW3&WsK&pSwj3H$VxcPC(Xy{mm&{F4|Hj}ZY?^y&Te0x+r3 z<(PQr)9;-ilB(OM3l8vtUH5^|r=L7S#QwQYuWfS-`}F0W-QPi<-u@yHSgk(Y>-kz- zBz|~iwT89t@)aUeZ0&Pfatzl#>s#YHL!Z9$1`+djeY*TKqX%$L==rm=7q^ZkMKkUR zX7#APV_JoxBh2-6we_U+?0Z5+qjPwFYd_i+-!o^SM1+W)7nywOycF8FwE@f8ehc?- zmbh4kWm;p(67h7&c(C*~ChFOiV%1o*w9B7@pF>SEB|=0k4*U6k^ z?|Fu0VmS3XuXVQS7r=6-B~j0|*bJqfPnT=uTL2c_wnT`iwCqhIm>CtF8V(Ea-R&~fWC6 zE-@^@-zm$f=cl3pEd2JH+owJl* zVOT2SsB3@m&GC5cx0^)Nvn}%uP;<=tOCL?Jl$aACqLvFM(9$bq>J_l`*-=*iBKKV7 zHHM|e!Z^Im99?iL9mXYdH&M^Fl%Ama^s9I6Yk;L+C=nuRi9d;!R5e*e=+hsW=9_bt z=$lpc>2D`fpQjI)biXrn*nOrFNg{o^#cFDAIpgKzS&;J7f}KJ(XVJ~5&(r6;Zs7%e zdfGf9``~o4DjhmSBPxMKK=KPE9leTyHD&0K7HP8B3LS)excPS)-jsfXB zeU5HdaqRo|ObuV~>8~FWnX>uxrLQ+PgE`^MnQxZTIQsT~<7gADMp|mN3zsZ0)6yuX ztI||0YdhPcs$H{5C+O=#E|YrkoKRfI_Cep6T=5&s3CXvJh}?Zqxy0^EQ`1VPfaURy z=8Gkkm}mjZ+?H&9^J}_$F0d54l6o=An)NB@JJzk3@Gr0gyAu&vmb2Hejkmw4>Ip2x z~ERSLZEV>ifHxSKqjyeI0Up%Q7vwU@DYv0>k?!^MjzPCg~mc{EjvW)k6^%_{J zxScsDvG~6fu$Xpd`$yI5#v205m1Cq{%wpViN))kfaS>xw!gx%(}C;2Qm&BHi&?%WGjE&L#sI%RTBfG==OScT_TEC4)8-+4 zfTe2Ym}rUR;Clg!y*Fd=sFaQWyMFkU)Qef>2M1%jJb8FZ720LjPa-1A;(QxfoVHF_ z3@rVIyQE1h&Y1$1ygBTB`ie=B4WL~XjV1MBmf=2ZjWbH~)J9-wZ%IUCS+@8gOG}#( zxUcfSKcNK@i(|HcWq4=iJLfMOFc4UNw;}an7F|2G{**c8(-vTv(uIh~vN+yBmP(a% ztiS{{Ue(?t@W1@DB}ZxIX!dVMGy=Kj+gm|eIJ%iA$z+cDh3xDVe@BN)pt8eRtU`_E zC5@`MHsILxS`uWAllzEDNtr`ei^ZwU$?u0iP?&m{$SKPlv07K~F15%pc@I(xjw_L( zkiL2Bt@qSsRck}$NI65K{wi}^)jdBJGKb4WA}OCae$Qu_qgR&%{H6QU8$?Lz;tH9g z!*nmaS0iTBz_Upb{e>d*E;TS`nPWo2?qiTS>_do188b&8!@X9JIVwLPVv3o=C(Rm@ zLc8SW_uz5c7EJ_{WRBC?CsApsv70Mo4xf0URNBm;aDn61)NVlLX!?f8mYg|yAM?Y^ zQC@pQE65zNX+*G8nWO8_ZP+n-X;;Q`!jrRzrtH`k%*MWy_s_OM<~a3@$doN}j4WEP z1?IP;>{)}PaWwafU{3Hp%Kou8dPZB^lb~Bb>cw-y%UNt~K(%4fI+znWl$-tMB4qE& zN`IUaG$-Gf0W8(mADAq$q<@ry_lti2K)9hWo6nzU}<4O zL}XbO1RzU(!>@7BE=kuaZYW*O9uoiV9ZRP`pb zi{@}5BFi%QKC;}&yIv344SQ$o-tQRxMAP&;sTZ@PTC+WGBR+n8 z3M|uq6A@XKu|ddED=+CPu+;4KBAD&*9V`jHJGTB*{4@Nx90_z>+2(koHC!Tm&C@G zFOD_)g4FhLN~9=cz`*bL*89Fw_d4Ly$4w(rf92E1CfN4{pT5ePNXq;4Zf}`SPw8!l zpXN?qK!l_&uHe)C;!dI+o8HBlZ!t_VJT@MH08!V#~e2Hnf6_G z9(?+@^F*d>KD}k~V|)+wI8mj(G>*FN5zGn0z1Z2-Teh!Kg*ibljns?h1h-Un-phj~ z7Itu7lCy}2+AX%Zf@-))Gsqr+~#Oh<%f{LhnHZV5v}p)QeeM z3K`4Vtop5hCAvNlk!9)d2w7fEOF*CA-ClE*#1ii%V7X_*^3$RDLHJ*=PislNnB`Qf z5X`kBMt#KpwdHOhBC;&4!jQ%6$_iw8vfk81VhKMZV43rcy@Re;Nxv?%OPB4WUd$4o z#Lmjq8|HN$+9iJv5s_v2=P|M*`1g;2K6W<8;kd*S=qq4xKhD-U;@<7(0xZ7^NWGY4 z>Pwb68XY-;-#^_|S^4K8WLX+NL6&#FibH|rV)&J763ewq0+wEBjHQYBMqKBp@`BWh zS+qv7aV%>0r*Y6OuM&xfEK9v`WZBzdQ3WtT>NjtHm00}#$MXM8(BkPf3))V%vRbxy zx|PHKt*HNWWSPURxUfB>g~fqHNhWh_9?dd`L0Bt1>Ho|_u?jVs9@cC>Qj6$WmmzaR zJtZn7WsZ&iu(g41FUH(}pm6U6kyDmAwyk3G@qwBF-#}_^q7o?zxiFZm4d^s4xCoh} zFoj6{Rpz)AJLUvr4v$PCDW5r3JY$*Ta-{)ip{{%(LQ)r3$Q&7KS>`a-w>=5ZynBIA zgx;l)cPyE`ThZ}4WR4NVM5K(Fqp;`dUywN(RP_7vv?ykd#`&u-Db!hft|>fjE3}D# zlFX4Fb_td04({g#nZv0%Q7UcbP`JS4&sm2cbC}m7vL$B@odIE(IW)Ir)`iRw*q8{G zDs%LkwhuGM=6|O7!PwWV8PSv-`}Ek@cc}UEHjp`@9Gj~HE_J*QEb||edNIo&JLWrH88@s2?NU9Oh{&>NJVO?n4)Y&F z9}7R^+*@MV6ewV+J)6CQZacp*uGxG(LF&aUkDQqw41M|QD6l*|OGIQ@%10v0jmMgA zfn{K?Fbj#LAXvcSG?%r@bfd}mziaOSq+ZN&bR%2mc$wbn3$XYNCnB;eg^Z=~44n?Z za`L^}Y>6f7p@8LDH}maebeIj7VI>y#GGjG~t!Sgb4CaD**)Y{L^6xy}icoeX#v?U_4ET5y1WmW9{ zDZn!5#ad^HCFYrc<^CP^yi6(R^cz_Gl1ROnWm_2guX4aDmmR=zKaGgUvSdC-mVrN- zrGs%eyl?npiDgfeQlH+TCHtf^sO@kc@af;(iIR*@zp?uaPQlA8KjJw&eZ0gf)STt) zVtaU|9Im7fKE3cPQ7Or%dzmwz?mVNmI+$y_D@0D2PmgKF<}fS6kAp$V;I!ZB6FW zT@JlT1ChKKp$NT8mMfS~e{`(TC-CVHUJ;Qp`t*sGVU@wB`@SP$iaz~a@dk9)lkx{A zz~j~=lL#pB>ASbwM5Q%*tnYzOPyS4lO6$`VE>O3kxf=NN^Z7)!WS@SudL;UESCw+^ z;M2{Eh+wIFdiMQ;I7n>lo^uh#z9h|m{|s7kW1pjW432%yKQ|YGPrp)?$dt{eM{0KS zfjOa5!Q5DB9DVd$FehZzV(*~$=qId!zW!2W-k*B$oG?Fu`Hsm`4&q*s%^E~R?!Ks` zvio9dWsm!y2TpL$kXZcV1S|!D{8^_rY5m}bk zFR+cn*0@)Nc4>Y2d7;E|>6L(G+5_f0E}YLf1nuH@o79V0Dp|7s=}b=k_XDsjxKBi6 zS>j`nWlDp%alqnUShZ^Ke_7D$jeteXKY`Wq1z2!F&Ib}Y5yaw~>8RgdK zf=@5EU5ONhjO2TGZW?d*0H5A@50UySpI#7O+Zud&h#QfV_vv<3I-`@CX6UI2UHrHQ z5t6#Nf=}-meF8<+KFz!YB1Wf#BJ?f|b!9$1Zq~*Y;M4VdiAWiJ`rf6L)WN6Uyh6kj zefk_VCv?|8TX*{okK0&3BA~>lkFyRyr6*}8w}VgL7)X>#>(do3u(r9%Xz=MP9}?M; zeY(-2IP~cSjh^X%Pd9r?1WV`BqmE+7NQ$z}gt2eo3!*7I_AN`qv2Rq4Adn6K> zvibBHnl0U6PKXVee@+@l)4vMlgs%L2u4Z}T@Eo4^5u{!`CwT2)zGG_aX#6+Oi5Emf z_P$(W_eHg+mkqEqYkn|5VoA&su=E;!9-GJWOVz{h-2P}!>cuRL7P9w)AIdwtf;Nus zMnq&;E+kqul*@h7pj|rU^)8TDZf#Ik+-tA9f#uq_y^L_Jb!aiE7qbjF$ex#=@rk%U z-A=vFpNo)Xaes>}7Ii`{0gKv`<&{GI%TK;r1S|!&eerqOp%eNI+GXc+QZHtS8O8co zgRwsjLA&&OMMPv-4kRPXhuJ-P153)OOARF!&+P)1_s+>J6-bMZ1{Pi&@&&XZtEI z`lsWWLN(M4|6GJD%cfLh30|9O0Ve3DspEQyCFFlBO7`#=TuQJD}>za)qb9`GyB;_;5tja8N zY@K-+m+{k96CtUKD`bxD$t-hBI8^QiJSWdL3PtE$3T(FZzm#U%p4Op zyKREZacmC}Q_LKJ8apv5)VlcV2Rv?$hlqfZ%%RsO6qVe5Jbn$CZ9t zJ~7}i+?P{BiHPidS(}D)!tAYPjxfLVzkBbL#Pa!&fJNW^9D1nQ_U<)+WmhPv7qe{6 zVedKzC5@Z`EG`j5M3!a6dt|v>oRkYJ;cN15Ni3;H1uUD^vVG7mQ`@xzmg8GUy_m&m zEc59(?;AA*mcA}TM3!ao2W0uww|z9QcueaUF0sTP7qFa)Wxhje-;Wo-5@bW_#VjLt zv9kB#b`We;3SYu|6t8;K?Slz^pS9NYJzJGe&^WEo28#ViIB z*j^D=JNr4%E+0k{5m}bm8OUO*%Ki&hYm4t!i6ziSz%sfU%TGz)9^<;;S{G6;X7My- z?P6Ksczs|=I6y>XS*B$oOK{k@=FrDR?@F%_`d@y!c3!~ptupgB87i-F^KPIUsTZ@n zaANK9a(B~2V2M9QL}XbeengftOEU1^bN(w0>PjqUE(=)B`m=KiVoo$X3w`Xz22w9( ziE7UFRjT_B#C5^P+lh!Q%jhg*dH%EBLof~}djwCCSdLy*>eKx$v$cWsdv4AJhix^0 zD9QMAz1i$s_Qg}}<3Xj*FtG|XXAdtipS~{Ls1f*dlhH(_B%f|Lnfde@%^zfdPk%p= z$SL#bd(>G!?iy`f6QnLoS0Y6rV-_=?{`ErcD}q2y13nPA|!Qj1)tvT6|-YD?b`MPk*{lnBJ?gzpT&Iot-zLzz^8xNL`2Hy z)7wpoiv*v3e+Lm$^y%HS_oBPDh)!PvkK46(do3 z&~{u-z;eyb-yl{6VmM1H-0PPTI)eycZ?_X;yIy> z6FXxv%zI>SxG($O5)rxkqB8sw&IvzFf_ni=QO*G zn8j}#J9B)>ki%KPvZN{zk!3N@MwS`}JF5eWhk=2!#IotGfF<}CJIA!s(v7&+{!1%T zFJ`IW&E9oB2{yp*209uO5m^?q9Aq)iY(E}Y9!;BkOkybr6tL8v#lBF7Vuh__j z)Qeeuer1`XnMTd7&@T7wh=?ppkI%@GyrMJu!I6QkS0t9K5CO{sRhBu*hfc!tWk-jR zdNIp^TkLr`@X!{&fl!YkBC;%Zqu6fHUK_9E>tC}FO#Doi2?)=L$xuVvvh0Q`WX#0>)wcV{PLH|K+Fr=SqEgcxUF*#}uMK5By(q2u zB{0`V(utfhpYHvgjV~)(j2jJ7@i|JQDCETb%%`i(nwJ4ST{oXd{gqF@lwo8EKK<2i zA}R0Fja8UWU)pzeZ|LHq)O7xIE2)bs_;mNX%%{gaaK`84s+LfM-lf>ld3ajU3QezX z;L}f4BO+z==@rLizXP9Mtqu`W^yz-p+|XV3x6$&)$E^_&P~y|KB}SuC@QQOP;M3O| z5~b4mbcGAl>6>~DeEN)5M7Cs~e&9tG`t(UNgA>80H|$6ROXbsBRA-w+stt~s9Z#y%U3dK1B?zwbk2%I4G0bs3@ubHa<2XP_k(M__u}~# zS{q5dcup`%WP4Ow4te+V!0GAU>Vbwt>X`@ z*K<5P{9P82dNIqJS)TY!?4Ta`yMU#+0}+vBsgs8+bwe9p02a?tqb(&CuS5aM&`vjS zZJ>Kl>T+P2Gnv$jS+rjrN0tFS>*4w7-ZP1aEQ@YFvQ&=CGzAua^}UNEmV<8vERm^h#6JF+^d7qg7{$^M;irdkyKx1?`m!9rcw+QOJ5;>>M7W4V&=)y>s-5 z)L&(ewV#7VLFTC6f=J3|js{vRb1Yk^ZwrrWYC9q%b#aBv@#Q(o92+95RR)pJEm~f!(tpb6EE$Vv3o=uonB9m-F4#>)~kxV%%M4oD3vyIC|sb&5d#Ov9Dx&vY{{9!BKaF;j-wIb!yt2*O(TM(${e|7 z?0|&jqwh9@u}^Cb(Ucqe9RK-^W1q|3O~W8_crG9^Wy>5t8ok4_KWn!; zOUN$)i}wNMJ318h#s6+k+ePZdEY8u)Lj^4l8VPNjeSnC_vb-z8HrAPFi@qb@Ub~;f za;sRta%3O-CU*7K?&t5-7S|mDmR4`s_wXOSbWMbIv0q5)#VqOGY@Ng9zQb~8mpRLc zh%C#ja@a0r1!?%dwl#axcS|fzy96v-quJ}AhpTHh2bN|>NWGXv-JOkNy|rw-fJOf# z5s_ti$5?6=nlFTQndRKjOJXV9D_}Wnz{at<;XWF`(t1Cs7qjfuX6;g#Ztxpeyxob2 zEX&mL*e-24Ue^cXPcGG#;0esV`~Eq+b0|W zm63DBD%6}^n8Pw?rUJBg&cPrq1XjIp{=YQRzG;%2Ty zNb2GWKHZ=^JBMe8nsy9`Bp()v(7QCqg6-j%7HwSxe0u6}B2q@5?sMk}?%`Q{hKMQp z^gZ>Cqr1Kn-aP^yH@!d8fY>e7gQ!B3rUg zcQN>dK7G;M953+c1;IqHR6gC`jZM;)<%hO`v9HDxqA5G}on>QR<9Trxz^B)VCNgF7 z=}#KlwuU*uYyR~w(m4A0fM8C@KhM?%enkw&x%&MwQZJqprtM(fc4|~@2e>cBYl(>L zeMw~Z<(Kx&55SV+o2vHczjrsOhXpLBN3yxPkyid z_lSrr%LG-N-`pBJ84WBtL+aF#SmHedEUy-@Jv{w()*1mU=A%fxn59E3o2z#@+Ejsd zF`Gn0WLaVvOZ$i8bb&=}*|3%pOZW)^i^U!02QR2CL7(ouhSZB$KI~@iZLRvO$NfmI zn~8`l%P2K$m$cX|slYPJXG1TECD2>I;J} zE*EYS5m}Z<#?pWI>q)@!dBm-e63ex-0v3z0?7i*En6LQ1f%y|iy_n_rdFE}C-n_)U zRR+_Eh%C#n3fL|)uP06imagBkXG$z*E(loeerGJMF2S{+T{iqC^d`K zb(NApfdA2XjmFE4&M_;mYvM5QF3?%I#}^i@N5lml~J+?dEI^XXSinNM%s zec5J^$}mzQMIk4zWe&7SZeK(2=`Gt5slW2+pF%&51fPE0m`KX|bf=o^efrHQF+bsP zZPJqnNnKpQr=RV_eEL$;dgnmoL|>r@y-U64vAcA%Ud0IT=>da@NEvA?#geyz1J@z!DKbL}Xb^)p34%Qgrb;uw1SeoG!5x+!e5_F3-+S-)cK- zB(S_|O6tWdhg_M5a?Wg3i7qf&{V0#i?uH1GU+GW@#A|lITqJizwqs>)Zf7)r;qDt6*S@3PBfMw7l_CBwx zO3h+u7iViyFJ>8cl;zs59ZuH)mVFC}h%C!3#-f+?^dhv&3iqi^C6<^l0gL5pwx|1k zEi=qd&%H>!nB}M)+ta=IP7OR`GS-)f$g&t~V!Mo7({m=UM0>j$ODvBf1T5=oGnVE( zE$c(OG`~pd#Vjja*f^&0qUl>`mkT$Eh%Cz$#$w`94ZlqZwt8$XvG_*`So)u3-^3Pa zF2r@gmbRo`%(AJFom~^sH1H#^bY4nCWLes3VY|$XTZ{kh_McchNn*M5LcsE*9-A`F z`aSLl?XusB)Qefn9Ufz@J$vSL2Cy8QM?_>M=Jk7tEof|lg=Ix4ZG#461k&1_ia z$aM+HgtTB&M3iJQN1st_Ecs@B`8cSouh9F?^$-QcLO^4?ad%_7*r-IC1s91 z8SU^*cUS)jaS#+HR3~!EGRF~Zw*6q5;X@6O%B`zJibAT*V`~GQ)C$}nb2Mo}r2Z;% z)J~ro44EU_h)Bw3j*7W#Z6IlN(g*0`v)d3Msf#OQ4#(>(b7(*RG#W&-jfEogE@hX` z$L|IL_IaB@<~VIaM9P>sMn3N40-2+V84**=9MkmAp#!&a|4-5k8}K{Q+`2?WmPNlZw#%>IYWUst_ZE{=Bo@as z0gL`V_O8>m!j;X?F7_Qsy_hBYJUe6ZQ%{>HXqV0=L`0Uwld-7n+y4XF#mQz*p2U)y zE?_yohJBmjnDvhtu>7-_)Qef>M6flR<^_TNz>?=cL}Xd?w6R^(jRV>OOP~CQnveg> zPw80#mT6bn8C(q~ZczgkHI+wy>cuQ}HU02;32D1;AF!xv5D{6H!;EFITjznm@-+E( z1BoT^vw)>p09#9Q%Q>40EWI*Gy_jY5BIeUS8?|}}ERJ7^h%8HO9c-7(Ul-zm#Z13L zJBcMSSHR-8gN?}z?>x){mQO87y_m(LAIs?Wj;$?#WoSntBFo~+SYpTY*a*fUB(mcY ziRE6tQlD<3%B-1V+WU6ku)ik|CE~E@JF?>;r*H~>UD; z`1F2-MCz}6`r#9UAAnDz%`t(pgi>BbyD>o!!ia!0O5&KiE$;7ev&)Yyl zBA~>l--^gZ|2xTAZwdJHw3bAvv_4(o0_~~|asi+ItOJoP*{2ugsN;23SGm*%eENuP zM6gso{o2`wI7p1F{b3M{eZBe+O}VizFo%tO1KeCHf={nBkjRwHr$?rATMcujF!b=gA!s%dJ8I%jcp1Y@U}Dt62eyjSs08vs4SXjVvpAkH>Y6h|5Gomc@m!B&;(s z2NsjU=N=M^ucM~on$6T}?Cctkf^#mwQl~=9pL#Ki=5N+6ZgKq6m&)Y$Ww!7Cl?m$Fwu^^PpYY?jRzvEETF@ zyOh7$!wULXgzD0GiN(oT!1D7uYnODbV|b25S|w61W*MErvfwd?-FQZSj~YZomSqED zsol1|5wHyN^!X^U6z&qR6xCxaE-^E-pk1z?A@yPwqkHVzl*Aqr@Oim*iHOLu6f>6Q zW^M6Y`}S9oiX@iL`vfefd$MoHdwE!_0v10%QZHul(Px>%v5G?&^sya5L`0TlwJ!Ft zayy(cqc=URtNTRSSLr5TSrp7NhtryhcEA#Nk<^P><~L_+X^q>z#&fFg+$18hEI$~F zc1$vU-y52mXfClFJ*cVdy8)-3YzxY>-K%mSE%eMLN-~+lHQ+eTS1Ca;9YAI5cd-gJ zn*D66nPWmBQ7I{Ny!*{EN6wdkdk_>HRFnSPBxRYS!C96$y8E2U1*sES zN~9>{`O_?OJjvOg3z;La8j<>|%<;%>mmXw}yLE`9eCGJyxf|Gww$2(}(8Y%|B0^FZ zSI8Xmio8(d-74K=5Ha{iC_?X&<1n^@QZ#c*W5^ukTN9BoW)9OHcjrUqSl5w=DQ1qg zt*)a3_Z?w29U^{B6C$7_bCg^43zd3$%x(;sqq!+jDsARaxWJ@VCRZSH^fxE6C1;L^ zE|u^)12R6{hRl&VoCuaGbA;E3z>cxGNiB64$G?psnsQ@bANA@ujxSF%6$6E zdh8uwwXPk1gVbSDB~lc!YCq=F&!;5U1)tt?5Rv*TpZ=oi6dmyCS^pAAd7u8V3-jrh zU!>x%BE!ZKA*qWi`1G3t**a0qsT1$vIhi>{C_?X2)ZMSRhi72(<(0vwH=0RA%IMS8 zIu3LOpZ<9+5mWT(XWIFpYkf7Zd?q|@uNM;mB|g1j!*X~iN6Ol|o4}|4>p+xB>(do3 zaBth&cfhB|uOqT0`*i!$I!N3lX4Wh4>5I1z!BYA3iMG)=)I8}Hoetx;+fJe>H}*L; ztc~ON?UXvI;L}^X5}C63^tOYyB*2{T^K*6!X&f!^6U+&<53+YRuln_#4}HDg4^l6l z6U>&fcWQHtuib|GvbJ2#pNo*YFDjPozEm8q9RMso^Xm1ISh4~HER6faUd4 zQZHr+U3U-X>P=2Mc>bTk8X_Xg@`|xce|_5zSa#SA8!5584HB@FtH}0O^ldN;|2NRU zfz*px!V4pirRL9BxbLOadLkmrGQJM>*L0^z9>C(0xM8Nm67xX7ayE&*w;ei670;Iq zOC|MUmiAu3$nxDg`2h5>uUSMymgNOwd0fv6zq>AOa(lVN^60UErT=H<2kY!FTm*g0 zx&^5hv()_=f-E;xU*g`Xf(}GPmc^njYnSwHYoJ|bre|%JSp1&~Si&M%AG20p*bi9l zd?xi`mcZZ4r@ysd6bkLKwSb7ovOHrfX7?sl0hVbIO+6%*OVI+B<*9)oI9L}XdQ7>nWT?!MqWPBy6=B(cONXtq#v9!AFZa2`Hb mXo2Se1-Bpqve(hMKHgKK9}m94oT<}+$dql)%zZh){{I22UjNMi literal 0 HcmV?d00001 diff --git a/logs/self_tuning/ademamix_golden/study_2/wmt_pytorch/trial_1/flags_0.json b/logs/self_tuning/ademamix_golden/study_2/wmt_pytorch/trial_1/flags_0.json new file mode 100644 index 00000000..159d9d39 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/wmt_pytorch/trial_1/flags_0.json @@ -0,0 +1,63 @@ +{ + "logtostderr": false, + "alsologtostderr": false, + "log_dir": "", + "v": 0, + "verbosity": 0, + "logger_levels": {}, + "stderrthreshold": "fatal", + "showprefixforinfo": true, + "run_with_pdb": false, + "pdb_post_mortem": false, + "pdb": false, + "run_with_profiling": false, + "profile_file": null, + "use_cprofile_for_profiling": true, + "only_check_args": false, + "op_conversion_fallback_to_while_loop": true, + "delta_threshold": 0.5, + "tt_check_filter": false, + "tt_single_core_summaries": false, + "runtime_oom_exit": true, + "hbm_oom_exit": true, + "test_srcdir": "", + "test_tmpdir": "/tmp/absl_testing", + "test_random_seed": 301, + "test_randomize_ordering_seed": "", + "xml_output_file": "", + "experimental_orbax_use_distributed_process_id": false, + "experimental_orbax_use_distributed_barrier": false, + "submission_path": "submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py", + "workload": "wmt", + "tuning_ruleset": "self", + "tuning_search_space": null, + "num_tuning_trials": 1, + "data_dir": "/data/wmt", + "imagenet_v2_data_dir": null, + "librispeech_tokenizer_vocab_path": "", + "framework": "pytorch", + "torch_compile": true, + "experiment_dir": "/experiment_runs", + "experiment_name": "submissions_a100/ademamix_golden/study_2", + "save_checkpoints": false, + "save_intermediate_checkpoints": true, + "resume_last_run": null, + "append_timestamp": false, + "use_wandb": false, + "profile": false, + "max_global_steps": null, + "overwrite": true, + "hparam_start_index": null, + "hparam_end_index": null, + "rng_seed": 16914702, + "set_pytorch_max_split_size": false, + "pytorch_eval_num_workers": 0, + "capture_jax_trace": false, + "skip_evals": false, + "?": false, + "help": false, + "helpshort": false, + "helpfull": false, + "helpxml": false, + "tfds_debug_list_dir": false +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_2/wmt_pytorch/trial_1/measurements.csv b/logs/self_tuning/ademamix_golden/study_2/wmt_pytorch/trial_1/measurements.csv new file mode 100644 index 00000000..f6e5372c --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/wmt_pytorch/trial_1/measurements.csv @@ -0,0 +1,458 @@ +global_step,grad_norm,loss,train/accuracy,train/loss,train/bleu,validation/accuracy,validation/loss,validation/bleu,validation/num_examples,test/accuracy,test/loss,test/bleu,test/num_examples,score,total_duration,accumulated_submission_time,accumulated_eval_time,accumulated_logging_time,preemption_count +0,0.4999999701976776,11.593880653381348,,,,,,,,,,,,,,,,, +1,,,0.0005829504149235,11.773135844592275,0.0,0.0004835649898947,11.785403311800223,0.0,3000.0,0.0007088489919237,11.775305037476032,0.0,3003.0,71.5164270401001,511.1090278625488,71.5164270401001,438.9007964134216,0.0,0.0 +1,0.4999999105930328,11.594104766845703,,,,,,,,,,,,,,,,, +2,0.4999999105930328,11.596843719482422,,,,,,,,,,,,,,,,, +3,0.4999999403953552,11.57876682281494,,,,,,,,,,,,,,,,, +4,0.4999999403953552,11.564610481262209,,,,,,,,,,,,,,,,, +5,0.4999999403953552,11.542754173278809,,,,,,,,,,,,,,,,, +6,0.4999999701976776,11.516008377075195,,,,,,,,,,,,,,,,, +7,0.4999999105930328,11.49023151397705,,,,,,,,,,,,,,,,, +8,0.4999998807907104,11.44612979888916,,,,,,,,,,,,,,,,, +9,0.4999999403953552,11.411049842834473,,,,,,,,,,,,,,,,, +10,0.4999998807907104,11.350812911987305,,,,,,,,,,,,,,,,, +11,0.499999850988388,11.312066078186035,,,,,,,,,,,,,,,,, +12,0.4999998807907104,11.257694244384766,,,,,,,,,,,,,,,,, +13,0.4999998807907104,11.213573455810549,,,,,,,,,,,,,,,,, +14,0.4999999105930328,11.148491859436035,,,,,,,,,,,,,,,,, +15,0.4999998807907104,11.082783699035645,,,,,,,,,,,,,,,,, +16,0.4999998807907104,11.031064987182615,,,,,,,,,,,,,,,,, +17,0.4999998807907104,10.974579811096191,,,,,,,,,,,,,,,,, +18,0.499999850988388,10.899060249328612,,,,,,,,,,,,,,,,, +19,0.4999999105930328,10.843337059020996,,,,,,,,,,,,,,,,, +20,0.4999998807907104,10.75318717956543,,,,,,,,,,,,,,,,, +21,0.4999998807907104,10.686498641967772,,,,,,,,,,,,,,,,, +22,0.4999998807907104,10.641940116882324,,,,,,,,,,,,,,,,, +23,0.499999850988388,10.562541961669922,,,,,,,,,,,,,,,,, +24,0.499999850988388,10.485610008239746,,,,,,,,,,,,,,,,, +25,0.499999850988388,10.41568374633789,,,,,,,,,,,,,,,,, +26,0.499999850988388,10.344508171081545,,,,,,,,,,,,,,,,, +27,0.499999850988388,10.282468795776367,,,,,,,,,,,,,,,,, +28,0.4999998211860657,10.206717491149902,,,,,,,,,,,,,,,,, +29,0.4999998211860657,10.143089294433594,,,,,,,,,,,,,,,,, +30,0.4999998211860657,10.06946086883545,,,,,,,,,,,,,,,,, +31,0.4999997615814209,10.003509521484377,,,,,,,,,,,,,,,,, +32,0.4999998211860657,9.907523155212402,,,,,,,,,,,,,,,,, +33,0.4999997913837433,9.851780891418455,,,,,,,,,,,,,,,,, +34,0.4999997913837433,9.803230285644531,,,,,,,,,,,,,,,,, +35,0.4999997317790985,9.725332260131836,,,,,,,,,,,,,,,,, +36,0.4999997317790985,9.688825607299805,,,,,,,,,,,,,,,,, +37,0.4999997019767761,9.616006851196287,,,,,,,,,,,,,,,,, +38,0.4999997317790985,9.574620246887209,,,,,,,,,,,,,,,,, +39,0.4999997019767761,9.527174949645996,,,,,,,,,,,,,,,,, +40,0.4999996721744537,9.450848579406738,,,,,,,,,,,,,,,,, +41,0.4999996423721313,9.405712127685549,,,,,,,,,,,,,,,,, +42,0.4999996125698089,9.355545997619627,,,,,,,,,,,,,,,,, +43,0.4999996125698089,9.273893356323242,,,,,,,,,,,,,,,,, +44,0.4999996125698089,9.244522094726562,,,,,,,,,,,,,,,,, +45,0.4999995529651642,9.213996887207031,,,,,,,,,,,,,,,,, +46,0.4999995231628418,9.134466171264648,,,,,,,,,,,,,,,,, +47,0.4999994337558746,9.118485450744627,,,,,,,,,,,,,,,,, +48,0.499999463558197,9.062204360961914,,,,,,,,,,,,,,,,, +49,0.4999994039535522,9.04249095916748,,,,,,,,,,,,,,,,, +50,0.4999993741512298,8.985867500305176,,,,,,,,,,,,,,,,, +51,0.4999993145465851,8.953688621520996,,,,,,,,,,,,,,,,, +52,0.4999993145465851,8.916143417358398,,,,,,,,,,,,,,,,, +53,0.4999991953372955,8.916048049926758,,,,,,,,,,,,,,,,, +54,0.4999992251396179,8.897092819213867,,,,,,,,,,,,,,,,, +55,0.4999991357326507,8.844551086425781,,,,,,,,,,,,,,,,, +56,0.4999990463256836,8.829573631286621,,,,,,,,,,,,,,,,, +57,0.4999990165233612,8.81917953491211,,,,,,,,,,,,,,,,, +58,0.497636079788208,8.778531074523926,,,,,,,,,,,,,,,,, +59,0.4690460860729217,8.77975845336914,,,,,,,,,,,,,,,,, +60,0.4488116800785064,8.706215858459473,,,,,,,,,,,,,,,,, +61,0.4165886342525482,8.715714454650879,,,,,,,,,,,,,,,,, +62,0.3915183246135711,8.717236518859863,,,,,,,,,,,,,,,,, +63,0.3927037417888641,8.648736000061035,,,,,,,,,,,,,,,,, +64,0.3703529238700866,8.667266845703125,,,,,,,,,,,,,,,,, +65,0.3473883867263794,8.619146347045898,,,,,,,,,,,,,,,,, +66,0.3277903199195862,8.585220336914062,,,,,,,,,,,,,,,,, +67,0.3173594772815704,8.598855018615723,,,,,,,,,,,,,,,,, +68,0.3049185574054718,8.603001594543457,,,,,,,,,,,,,,,,, +69,0.2903312742710113,8.590513229370117,,,,,,,,,,,,,,,,, +70,0.2926087081432342,8.53958797454834,,,,,,,,,,,,,,,,, +71,0.2857502996921539,8.549455642700195,,,,,,,,,,,,,,,,, +72,0.2633227407932281,8.566936492919922,,,,,,,,,,,,,,,,, +73,0.2702756822109222,8.50511646270752,,,,,,,,,,,,,,,,, +74,0.2703572809696197,8.473642349243164,,,,,,,,,,,,,,,,, +75,0.2803642749786377,8.536870002746582,,,,,,,,,,,,,,,,, +76,0.2469469308853149,8.519988059997559,,,,,,,,,,,,,,,,, +77,0.2464103698730468,8.491155624389648,,,,,,,,,,,,,,,,, +78,0.2988972067832947,8.449273109436035,,,,,,,,,,,,,,,,, +79,0.228014662861824,8.492180824279785,,,,,,,,,,,,,,,,, +80,0.2361658811569214,8.495820999145508,,,,,,,,,,,,,,,,, +81,0.2276259213685989,8.500658988952637,,,,,,,,,,,,,,,,, +82,0.2422211617231369,8.385904312133789,,,,,,,,,,,,,,,,, +83,0.22237528860569,8.470479965209961,,,,,,,,,,,,,,,,, +84,0.2131891995668411,8.437685012817383,,,,,,,,,,,,,,,,, +85,0.2438833564519882,8.453454971313477,,,,,,,,,,,,,,,,, +86,0.2244519591331482,8.426950454711914,,,,,,,,,,,,,,,,, +87,0.2107634246349334,8.409098625183105,,,,,,,,,,,,,,,,, +88,0.2183837741613388,8.394569396972656,,,,,,,,,,,,,,,,, +89,0.2186059951782226,8.40248966217041,,,,,,,,,,,,,,,,, +90,0.2280148118734359,8.399572372436523,,,,,,,,,,,,,,,,, +91,0.2126075178384781,8.400919914245605,,,,,,,,,,,,,,,,, +92,0.2103816568851471,8.391845703125,,,,,,,,,,,,,,,,, +93,0.2067849785089492,8.375786781311035,,,,,,,,,,,,,,,,, +94,0.2129890918731689,8.376073837280273,,,,,,,,,,,,,,,,, +95,0.1988153308629989,8.40665054321289,,,,,,,,,,,,,,,,, +96,0.2215000689029693,8.329667091369629,,,,,,,,,,,,,,,,, +97,0.2114602029323578,8.338883399963379,,,,,,,,,,,,,,,,, +98,0.2213921844959259,8.337302207946777,,,,,,,,,,,,,,,,, +99,0.1979315429925918,8.361783981323242,,,,,,,,,,,,,,,,, +100,0.2022898942232132,8.328301429748535,,,,,,,,,,,,,,,,, +500,0.4999990463256836,6.6917266845703125,,,,,,,,,,,,,,,,, +1000,0.4999992549419403,5.5475263595581055,,,,,,,,,,,,,,,,, +1500,0.4999991655349731,4.681185722351074,,,,,,,,,,,,,,,,, +2000,0.4999991059303283,3.9243977069854736,,,,,,,,,,,,,,,,, +2500,0.4999991357326507,3.3987247943878174,,,,,,,,,,,,,,,,, +3000,0.4386123418807983,3.12247633934021,,,,,,,,,,,,,,,,, +3500,0.4251218438148498,2.880624771118164,,,,,,,,,,,,,,,,, +4000,0.393195241689682,2.832956314086914,,,,,,,,,,,,,,,,, +4166,,,0.5465434516243545,2.613605328248996,25.52435517381768,0.5516484606514488,2.552473388426678,21.09654901711181,3000.0,0.551833129975016,2.5637898727557955,19.5996366388383,3003.0,713.5417437553406,1352.306256055832,713.5417437553406,635.3937435150146,0.1140470504760742,0.0 +4500,0.3759811520576477,2.686922311782837,,,,,,,,,,,,,,,,, +5000,0.3316928744316101,2.481339931488037,,,,,,,,,,,,,,,,, +5500,0.3287293910980224,2.5366694927215576,,,,,,,,,,,,,,,,, +6000,0.2968069911003113,2.331198215484619,,,,,,,,,,,,,,,,, +6500,0.2945584654808044,2.3835787773132324,,,,,,,,,,,,,,,,, +7000,0.2946372926235199,2.532318592071533,,,,,,,,,,,,,,,,, +7500,0.2737062275409698,2.277169942855835,,,,,,,,,,,,,,,,, +8000,0.2480931878089904,2.258150100708008,,,,,,,,,,,,,,,,, +8346,,,0.5981583541573957,2.107146575725419,28.964552267912858,0.6117965059329704,2.0157274863299897,24.563140508826766,3000.0,0.6157806054267619,1.982142597757248,23.70812091842948,3003.0,1355.612999916077,2205.5780696868896,1355.612999916077,844.0106911659241,0.2689299583435058,0.0 +8500,0.2662537395954132,2.2400591373443604,,,,,,,,,,,,,,,,, +9000,0.2413852214813232,2.1760003566741943,,,,,,,,,,,,,,,,, +9500,0.2383093982934951,2.184194803237915,,,,,,,,,,,,,,,,, +10000,0.2259019166231155,2.148994445800781,,,,,,,,,,,,,,,,, +10500,0.2170709818601608,2.1472787857055664,,,,,,,,,,,,,,,,, +11000,0.2225727140903473,2.14050030708313,,,,,,,,,,,,,,,,, +11500,0.2024378329515457,2.094437599182129,,,,,,,,,,,,,,,,, +12000,0.2089239209890365,2.150453805923462,,,,,,,,,,,,,,,,, +12500,0.2075115740299224,2.101534128189087,,,,,,,,,,,,,,,,, +12528,,,0.6432607033202835,1.76966246447445,31.310136574467155,0.6332469529206085,1.830990161312321,26.32728614427261,3000.0,0.6398117483005055,1.7749698593922492,25.34061019767668,3003.0,1997.7971255779264,3048.332426071167,1997.7971255779264,1042.130030155182,0.3031799793243408,0.0 +13000,0.1979088336229324,2.053950071334839,,,,,,,,,,,,,,,,, +13500,0.1948316842317581,2.1136837005615234,,,,,,,,,,,,,,,,, +14000,0.2024207711219787,2.064310073852539,,,,,,,,,,,,,,,,, +14500,0.1865918040275573,1.9615758657455444,,,,,,,,,,,,,,,,, +15000,0.1976753622293472,1.9778704643249512,,,,,,,,,,,,,,,,, +15500,0.1813472658395767,2.003346443176269,,,,,,,,,,,,,,,,, +16000,0.1706434041261673,1.9273587465286253,,,,,,,,,,,,,,,,, +16500,0.1736134439706802,2.069386482238769,,,,,,,,,,,,,,,,, +16707,,,0.6300413768017097,1.84406188901014,30.73685338788265,0.6441953602559175,1.7323150053936096,27.362542756722107,3000.0,0.654546510952298,1.6592146737551563,26.680943990594606,3003.0,2639.9789066314697,3887.407883405685,2639.9789066314697,1236.5729298591614,0.336212158203125,0.0 +17000,0.1674244552850723,2.0455265045166016,,,,,,,,,,,,,,,,, +17500,0.1792759597301483,1.9821219444274905,,,,,,,,,,,,,,,,, +18000,0.181269571185112,1.91903018951416,,,,,,,,,,,,,,,,, +18500,0.185858741402626,1.9351046085357664,,,,,,,,,,,,,,,,, +19000,0.1587040722370147,1.9006837606430047,,,,,,,,,,,,,,,,, +19500,0.1592761725187301,1.8646278381347656,,,,,,,,,,,,,,,,, +20000,0.1614420562982559,1.9624000787734983,,,,,,,,,,,,,,,,, +20500,0.159108892083168,1.887191653251648,,,,,,,,,,,,,,,,, +20890,,,0.639524875193665,1.760459882653354,30.87344944228189,0.6528375345624977,1.6656911414613582,27.87425041864447,3000.0,0.6632967288362094,1.5971380948230782,27.197294463741816,3003.0,3282.243233919144,4745.681620597839,3282.243233919144,1450.1309490203855,0.369384765625,0.0 +21000,0.1584613621234893,1.8071327209472656,,,,,,,,,,,,,,,,, +21500,0.1665437817573547,1.8794264793396,,,,,,,,,,,,,,,,, +22000,0.1514926999807357,1.8849714994430544,,,,,,,,,,,,,,,,, +22500,0.1474206745624542,1.865800738334656,,,,,,,,,,,,,,,,, +23000,0.1471511572599411,1.8605419397354128,,,,,,,,,,,,,,,,, +23500,0.1492020785808563,1.767670750617981,,,,,,,,,,,,,,,,, +24000,0.1452879309654235,1.84420645236969,,,,,,,,,,,,,,,,, +24500,0.1402076184749603,1.7860864400863647,,,,,,,,,,,,,,,,, +25000,0.1623504310846328,1.7933733463287354,,,,,,,,,,,,,,,,, +25071,,,0.657928301371111,1.637032452853824,32.65569117177226,0.6585907180320145,1.6224818198162454,28.51985047537219,3000.0,0.6708151763407123,1.542599209807681,27.78879918576069,3003.0,3924.4118206501007,5602.4192237854,3924.4118206501007,1662.2493290901184,0.4033761024475097,0.0 +25500,0.1459185779094696,1.8047919273376465,,,,,,,,,,,,,,,,, +26000,0.1558799296617508,1.80181097984314,,,,,,,,,,,,,,,,, +26500,0.1543875336647033,1.8922150135040283,,,,,,,,,,,,,,,,, +27000,0.1831191033124923,1.8438855409622192,,,,,,,,,,,,,,,,, +27500,0.1564004570245742,1.77756929397583,,,,,,,,,,,,,,,,, +28000,0.1601996421813964,1.819470882415772,,,,,,,,,,,,,,,,, +28500,0.1527040898799896,1.831330418586731,,,,,,,,,,,,,,,,, +29000,0.1996507942676544,1.7509974241256714,,,,,,,,,,,,,,,,, +29252,,,0.6512682311847234,1.6789634393180584,31.91883672766166,0.6644058970130563,1.5817602618070452,28.512950924444468,3000.0,0.6761141130672245,1.5004787998954157,28.09399740478748,3003.0,4566.620882987976,6504.546481370926,4566.620882987976,1919.713871717453,0.4384465217590332,0.0 +29500,0.1710467338562011,1.7781882286071775,,,,,,,,,,,,,,,,, +30000,0.1695147603750229,1.823364496231079,,,,,,,,,,,,,,,,, +30500,0.1792331337928772,1.722183108329773,,,,,,,,,,,,,,,,, +31000,0.1566316038370132,1.7480648756027222,,,,,,,,,,,,,,,,, +31500,0.1558705866336822,1.7543822526931765,,,,,,,,,,,,,,,,, +32000,0.152761623263359,1.7901568412780762,,,,,,,,,,,,,,,,, +32500,0.1589823216199875,1.679573655128479,,,,,,,,,,,,,,,,, +33000,0.1822048723697662,1.7605825662612915,,,,,,,,,,,,,,,,, +33433,,,0.655044324422871,1.643601800893325,32.472740982188675,0.668435605262179,1.550556274100755,28.778562761376325,3000.0,0.6795886351751786,1.4645453852187555,28.33979120931825,3003.0,5208.822647094727,7369.7161860466,5208.822647094727,2140.2251958847046,0.4747128486633301,0.0 +33500,0.1644811630249023,1.7338203191757202,,,,,,,,,,,,,,,,, +34000,0.1680420935153961,1.763649344444275,,,,,,,,,,,,,,,,, +34500,0.1707319468259811,1.701491117477417,,,,,,,,,,,,,,,,, +35000,0.1757835298776626,1.808377265930176,,,,,,,,,,,,,,,,, +35500,0.2115088403224945,1.7636712789535522,,,,,,,,,,,,,,,,, +36000,0.1858585178852081,1.7388343811035156,,,,,,,,,,,,,,,,, +36500,0.1776649802923202,1.701938271522522,,,,,,,,,,,,,,,,, +37000,0.1858661919832229,1.788103461265564,,,,,,,,,,,,,,,,, +37500,0.2216990143060684,1.6487627029418943,,,,,,,,,,,,,,,,, +37614,,,0.667604986499684,1.553395903946688,33.290601624517265,0.6705186544494178,1.5284733760275755,28.94812452196805,3000.0,0.6837603857997792,1.4456154494218814,28.645861782241727,3003.0,5850.994255065918,8247.104498386383,5850.994255065918,2372.986330986023,0.5110347270965576,0.0 +38000,0.1829511374235153,1.796694040298462,,,,,,,,,,,,,,,,, +38500,0.197963684797287,1.6805213689804075,,,,,,,,,,,,,,,,, +39000,0.2053864151239395,1.7752749919891355,,,,,,,,,,,,,,,,, +39500,0.1980751007795334,1.804226040840149,,,,,,,,,,,,,,,,, +40000,0.1859464645385742,1.6827865839004517,,,,,,,,,,,,,,,,, +40500,0.2032764256000518,1.748924970626831,,,,,,,,,,,,,,,,, +41000,0.2304434329271316,1.6873148679733276,,,,,,,,,,,,,,,,, +41500,0.1952470988035202,1.6384152173995972,,,,,,,,,,,,,,,,, +41794,,,0.6589405104958028,1.6103582696447487,32.836647836182514,0.6729612776035016,1.5113276455034657,29.203161052740487,3000.0,0.6867236069955261,1.4221527112021382,29.011979674353288,3003.0,6493.1392595767975,9214.9997112751,6493.1392595767975,2696.274794101715,0.5483815670013428,0.0 +42000,0.1933686584234237,1.6829177141189575,,,,,,,,,,,,,,,,, +42500,0.2330166101455688,1.6045411825180054,,,,,,,,,,,,,,,,, +43000,0.2273343801498413,1.7420374155044556,,,,,,,,,,,,,,,,, +43500,0.2210697531700134,1.627815842628479,,,,,,,,,,,,,,,,, +44000,0.2114789634943008,1.7461775541305542,,,,,,,,,,,,,,,,, +44500,0.2112343311309814,1.6715623140335083,,,,,,,,,,,,,,,,, +45000,0.2093982845544815,1.666886329650879,,,,,,,,,,,,,,,,, +45500,0.2061104029417038,1.6536667346954346,,,,,,,,,,,,,,,,, +45974,,,0.6654793370616996,1.558517577393849,33.60134517601314,0.6750443267907403,1.4944295095535085,29.11852423321864,3000.0,0.6901400267270932,1.4036893193306603,29.25787622025507,3003.0,7135.297115802765,10116.984761714935,7135.297115802765,2953.6487753391266,0.5850110054016113,0.0 +46000,0.2375523149967193,1.621321201324463,,,,,,,,,,,,,,,,, +46500,0.2342200279235839,1.7238773107528689,,,,,,,,,,,,,,,,, +47000,0.2328795194625854,1.6295205354690552,,,,,,,,,,,,,,,,, +47500,0.2193128764629364,1.577849268913269,,,,,,,,,,,,,,,,, +48000,0.2735250294208526,1.7201590538024902,,,,,,,,,,,,,,,,, +48500,0.2355912774801254,1.6336908340454102,,,,,,,,,,,,,,,,, +49000,0.2189806699752807,1.615947961807251,,,,,,,,,,,,,,,,, +49500,0.2162935435771942,1.6904598474502563,,,,,,,,,,,,,,,,, +50000,0.260880708694458,1.580472111701965,,,,,,,,,,,,,,,,, +50155,,,0.6773328180237118,1.4851885486171894,34.27266118762908,0.6784292817200035,1.4790820363355692,29.64243472164039,3000.0,0.6913834175817791,1.3904068437336587,29.11615660221508,3003.0,7777.399891376495,10994.264209508896,7777.399891376495,3186.342133998871,0.6584129333496094,0.0 +50500,0.2364303022623062,1.6164820194244385,,,,,,,,,,,,,,,,, +51000,0.2504015266895294,1.6236861944198608,,,,,,,,,,,,,,,,, +51500,0.2671704292297363,1.6992132663726809,,,,,,,,,,,,,,,,, +52000,0.2537094950675964,1.6125046014785769,,,,,,,,,,,,,,,,, +52500,0.2450892329216003,1.6313118934631348,,,,,,,,,,,,,,,,, +53000,0.2465381026268005,1.671269416809082,,,,,,,,,,,,,,,,, +53500,0.278950959444046,1.615919589996338,,,,,,,,,,,,,,,,, +54000,0.2560375332832336,1.6887388229370115,,,,,,,,,,,,,,,,, +54333,,,0.6660078153398764,1.5606366860914214,33.61155742324531,0.6800411650196526,1.4692231032473249,29.7080990770268,3000.0,0.6941723316483644,1.3778477427226774,29.43885734242232,3003.0,8419.607409238815,11936.813574552536,8419.607409238815,3484.241970062256,0.6958703994750977,0.0 +54500,0.2670340538024902,1.5218656063079834,,,,,,,,,,,,,,,,, +55000,0.2507281899452209,1.571427822113037,,,,,,,,,,,,,,,,, +55500,0.3334909379482269,1.641982913017273,,,,,,,,,,,,,,,,, +56000,0.2918515801429748,1.6339150667190552,,,,,,,,,,,,,,,,, +56500,0.2872582674026489,1.5443953275680542,,,,,,,,,,,,,,,,, +57000,0.2807862758636474,1.6256675720214844,,,,,,,,,,,,,,,,, +57500,0.3015496134757995,1.6351464986801147,,,,,,,,,,,,,,,,, +58000,0.2605289220809936,1.610477328300476,,,,,,,,,,,,,,,,, +58500,0.289347231388092,1.591109275817871,,,,,,,,,,,,,,,,, +58511,,,0.6746003470745751,1.4995030376208154,34.61035100910041,0.681020694101747,1.4591866111393534,29.62571324524645,3000.0,0.695671372959154,1.3631363118064028,29.32969463228488,3003.0,9061.798766613008,12894.345242500303,9061.798766613008,3797.135009288788,0.7342274188995361,0.0 +59000,0.2889527976512909,1.6502684354782104,,,,,,,,,,,,,,,,, +59500,0.3352183997631073,1.6331626176834106,,,,,,,,,,,,,,,,, +60000,0.3263132274150848,1.6169248819351196,,,,,,,,,,,,,,,,, +60500,0.3455554246902466,1.6418423652648926,,,,,,,,,,,,,,,,, +61000,0.3050102889537811,1.6153981685638428,,,,,,,,,,,,,,,,, +61500,0.3017012476921081,1.595123052597046,,,,,,,,,,,,,,,,, +62000,0.315574586391449,1.720388650894165,,,,,,,,,,,,,,,,, +62500,0.3079628348350525,1.6686540842056274,,,,,,,,,,,,,,,,, +62687,,,0.6824528540400152,1.450200463702495,34.866682799736104,0.6835005145627456,1.4477991980880585,29.770633461405943,3000.0,0.6993666840973796,1.3493543373423973,29.72264275714696,3003.0,9703.985414505005,13762.728229761124,9703.985414505005,4020.878353357315,0.7698471546173096,0.0 +63000,0.3062782883644104,1.6054164171218872,,,,,,,,,,,,,,,,, +63500,0.3631040751934051,1.5700759887695312,,,,,,,,,,,,,,,,, +64000,0.3050289154052734,1.609994411468506,,,,,,,,,,,,,,,,, +64500,0.3318484127521515,1.5897130966186523,,,,,,,,,,,,,,,,, +65000,0.3186927437782287,1.562523603439331,,,,,,,,,,,,,,,,, +65500,0.3299461007118225,1.5452167987823486,,,,,,,,,,,,,,,,, +66000,0.3485345244407654,1.535647988319397,,,,,,,,,,,,,,,,, +66500,0.2960192859172821,1.5459553003311155,,,,,,,,,,,,,,,,, +66864,,,0.6756834169702802,1.4924265777880892,34.11392448626766,0.6843684517240952,1.4377314176203644,29.95421232958789,3000.0,0.6993434431468247,1.3424210897100692,29.931385894568795,3003.0,10346.117608308792,14712.024676322935,10346.117608308792,4325.582231521606,0.8068163394927979,0.0 +67000,0.3108240365982055,1.641825795173645,,,,,,,,,,,,,,,,, +67500,0.341688722372055,1.5409119129180908,,,,,,,,,,,,,,,,, +68000,0.3415662348270416,1.493855357170105,,,,,,,,,,,,,,,,, +68500,0.3103118240833282,1.5926246643066406,,,,,,,,,,,,,,,,, +69000,0.3314449191093445,1.579581379890442,,,,,,,,,,,,,,,,, +69500,0.3115928471088409,1.5382652282714844,,,,,,,,,,,,,,,,, +70000,0.3008898496627807,1.4874992370605469,,,,,,,,,,,,,,,,, +70500,0.3316295444965362,1.5409154891967771,,,,,,,,,,,,,,,,, +71000,0.3388686776161194,1.5587012767791748,,,,,,,,,,,,,,,,, +71037,,,0.6834053912230068,1.4317170691641663,35.21908852731881,0.6867242811620439,1.428437515498878,30.27562602585876,3000.0,0.7004009063970716,1.3316930306199524,29.98900185874826,3003.0,10988.23679113388,15632.579575777054,10988.23679113388,4601.567341327667,0.8427951335906982,0.0 +71500,0.3331415355205536,1.4721815586090088,,,,,,,,,,,,,,,,, +72000,0.3166785836219787,1.530073881149292,,,,,,,,,,,,,,,,, +72500,0.3280566930770874,1.5981765985488892,,,,,,,,,,,,,,,,, +73000,0.3329830765724182,1.5922541618347168,,,,,,,,,,,,,,,,, +73500,0.3363258242607116,1.5333234071731567,,,,,,,,,,,,,,,,, +74000,0.3109143674373626,1.556001901626587,,,,,,,,,,,,,,,,, +74500,0.3425824344158172,1.6656813621520996,,,,,,,,,,,,,,,,, +75000,0.3414843082427978,1.5539382696151731,,,,,,,,,,,,,,,,, +75218,,,0.688193824532491,1.4156583858351073,34.85227405520313,0.6874062317888185,1.4217455843696916,30.21420412657871,3000.0,0.7021672186392424,1.321363063593051,30.028664457576262,3003.0,11630.491575241089,16502.620292186737,11630.491575241089,4826.906831741333,0.8794615268707275,0.0 +75500,0.3705296218395233,1.5104819536209106,,,,,,,,,,,,,,,,, +76000,0.3523108661174774,1.5506938695907593,,,,,,,,,,,,,,,,, +76500,0.3530897498130798,1.5201436281204224,,,,,,,,,,,,,,,,, +77000,0.3357428014278412,1.5352703332901,,,,,,,,,,,,,,,,, +77500,0.3220401406288147,1.5192886590957642,,,,,,,,,,,,,,,,, +78000,0.3773557841777801,1.576794981956482,,,,,,,,,,,,,,,,, +78500,0.3188943266868591,1.5464129447937012,,,,,,,,,,,,,,,,, +79000,0.3700280487537384,1.5248358249664309,,,,,,,,,,,,,,,,, +79401,,,0.686185196964878,1.428450612533509,34.957344012463935,0.6875674201187834,1.4141449346567307,30.02909519582649,3000.0,0.7039451513566904,1.3115709066875836,30.37841595753352,3003.0,12272.63627243042,17356.511085510254,12272.63627243042,5036.204295396805,0.9161226749420166,0.0 +79500,0.3473791480064392,1.4517611265182495,,,,,,,,,,,,,,,,, +80000,0.3669358491897583,1.5064234733581543,,,,,,,,,,,,,,,,, +80500,0.3672968447208404,1.5517696142196655,,,,,,,,,,,,,,,,, +81000,0.3378964364528656,1.5017099380493164,,,,,,,,,,,,,,,,, +81500,0.3449541628360748,1.4824479818344116,,,,,,,,,,,,,,,,, +82000,0.3691104352474212,1.452035665512085,,,,,,,,,,,,,,,,, +82500,0.3276960551738739,1.4967191219329834,,,,,,,,,,,,,,,,, +83000,0.3326477110385895,1.479841947555542,,,,,,,,,,,,,,,,, +83500,0.3424006104469299,1.4668951034545898,,,,,,,,,,,,,,,,, +83587,,,0.6951212514202426,1.3669021962115386,35.89579936838176,0.6879641913925432,1.4048900703339078,30.144622406580115,3000.0,0.7051885422113764,1.3016268665388415,30.36694753249224,3003.0,12914.87961435318,18228.025629758835,12914.87961435318,5263.015841007233,0.9543108940124512,0.0 +84000,0.3490352928638458,1.495434045791626,,,,,,,,,,,,,,,,, +84500,0.3535842299461365,1.5015438795089722,,,,,,,,,,,,,,,,, +85000,0.3382699489593506,1.5008418560028076,,,,,,,,,,,,,,,,, +85500,0.3590934872627258,1.4754807949066162,,,,,,,,,,,,,,,,, +86000,0.3247794806957245,1.479132890701294,,,,,,,,,,,,,,,,, +86500,0.3057251870632171,1.3779200315475464,,,,,,,,,,,,,,,,, +87000,0.4096120297908783,1.3856054544448853,,,,,,,,,,,,,,,,, +87500,0.3339238464832306,1.4451371431350708,,,,,,,,,,,,,,,,, +87773,,,0.6960266867731396,1.3705178130783466,36.28646213688107,0.6907291912065566,1.402832594295173,30.39228775289873,3000.0,0.7062692464121783,1.2978454912555923,30.380732761160026,3003.0,13557.02722811699,19095.119864702225,13557.02722811699,5485.502369880676,0.991753101348877,0.0 +88000,0.3634322881698608,1.476938009262085,,,,,,,,,,,,,,,,, +88500,0.3497765362262726,1.47718346118927,,,,,,,,,,,,,,,,, +89000,0.3558787107467651,1.398590087890625,,,,,,,,,,,,,,,,, +89500,0.3364595174789428,1.4269509315490725,,,,,,,,,,,,,,,,, +90000,0.3512241244316101,1.4763448238372805,,,,,,,,,,,,,,,,, +90500,0.36087167263031,1.4764286279678345,,,,,,,,,,,,,,,,, +91000,0.3371688723564148,1.482085943222046,,,,,,,,,,,,,,,,, +91500,0.3333888351917267,1.4094544649124146,,,,,,,,,,,,,,,,, +91951,,,0.6927327011290452,1.3787650031743672,35.77508179254608,0.6903944154443218,1.3989227117456695,30.24663058347865,3000.0,0.7078728720004648,1.290620642321771,30.569424982767647,3003.0,14199.259328126907,19981.557898521423,14199.259328126907,5727.248344421387,1.0326135158538818,0.0 +92000,0.3359145820140838,1.543302059173584,,,,,,,,,,,,,,,,, +92500,0.4443472921848297,1.4075748920440674,,,,,,,,,,,,,,,,, +93000,0.3429942727088928,1.510561227798462,,,,,,,,,,,,,,,,, +93500,0.3557983934879303,1.4158402681350708,,,,,,,,,,,,,,,,, +94000,0.3881115615367889,1.4196068048477173,,,,,,,,,,,,,,,,, +94500,0.3376614153385162,1.4674619436264038,,,,,,,,,,,,,,,,, +95000,0.382350355386734,1.466017484664917,,,,,,,,,,,,,,,,, +95500,0.3489479422569275,1.4407031536102295,,,,,,,,,,,,,,,,, +96000,0.325073778629303,1.400094747543335,,,,,,,,,,,,,,,,, +96134,,,0.7004265985948693,1.3358186510400496,36.00848370991645,0.6915599310609912,1.390955707306791,30.495445705351244,3000.0,0.7087095462204405,1.2842161263145662,30.535210420138235,3003.0,14841.415359258652,20870.82671952248,14841.415359258652,5971.905826330185,1.0726385116577148,0.0 +96500,0.3640583157539367,1.4579360485076904,,,,,,,,,,,,,,,,, +97000,0.3757734596729278,1.4994680881500244,,,,,,,,,,,,,,,,, +97500,0.3770154118537903,1.528436303138733,,,,,,,,,,,,,,,,, +98000,0.3577263057231903,1.4422272443771362,,,,,,,,,,,,,,,,, +98500,0.3477201461791992,1.3890889883041382,,,,,,,,,,,,,,,,, +99000,0.3304465115070343,1.3934496641159058,,,,,,,,,,,,,,,,, +99500,0.3386832475662231,1.3764272928237915,,,,,,,,,,,,,,,,, +100000,0.337370902299881,1.4299880266189575,,,,,,,,,,,,,,,,, +100319,,,0.7012888725804972,1.3318683850678836,36.78394909790258,0.6915599310609912,1.3895603239575456,30.66003052684969,3000.0,0.7104526175120562,1.2800209531694846,30.594078116323715,3003.0,15483.591715574265,21737.572647094727,15483.591715574265,6194.022545337677,1.1121065616607666,0.0 +100500,0.3533065021038055,1.489389181137085,,,,,,,,,,,,,,,,, +101000,0.3459774553775787,1.3874976634979248,,,,,,,,,,,,,,,,, +101500,0.3346822559833526,1.4165219068527222,,,,,,,,,,,,,,,,, +102000,0.3251266181468963,1.509474754333496,,,,,,,,,,,,,,,,, +102500,0.326145589351654,1.3653072118759155,,,,,,,,,,,,,,,,, +103000,0.3831531107425689,1.4407271146774292,,,,,,,,,,,,,,,,, +103500,0.318098783493042,1.4058656692504885,,,,,,,,,,,,,,,,, +104000,0.3395566642284393,1.465869426727295,,,,,,,,,,,,,,,,, +104500,0.3346026539802551,1.4679334163665771,,,,,,,,,,,,,,,,, +104506,,,0.7011867710934027,1.3307043175781206,35.85118838233687,0.6919938996416659,1.3865825051766252,30.470427481360787,3000.0,0.7101969670559526,1.277310477311022,30.73572516837829,3003.0,16125.781217098236,22628.952724456787,16125.781217098236,6440.756788730621,1.1513752937316897,0.0 +105000,0.3305352926254272,1.3662471771240234,,,,,,,,,,,,,,,,, +105500,0.3232753276824951,1.3853230476379397,,,,,,,,,,,,,,,,, +106000,0.3387472927570343,1.467509627342224,,,,,,,,,,,,,,,,, +106500,0.3174803256988525,1.3937405347824097,,,,,,,,,,,,,,,,, +107000,0.3584260046482086,1.464188575744629,,,,,,,,,,,,,,,,, +107500,0.3357649147510528,1.368948459625244,,,,,,,,,,,,,,,,, +108000,0.3273093104362488,1.458005428314209,,,,,,,,,,,,,,,,, +108500,0.3296625912189483,1.3963547945022583,,,,,,,,,,,,,,,,, +108689,,,0.7037840941716911,1.316916271892047,36.14055865113249,0.6925146619384757,1.3840584160456784,30.54382365888488,3000.0,0.7104758584626111,1.2740839797222707,30.683772847933387,3003.0,16767.940527915955,23491.281604528427,16767.940527915955,6658.462338924408,1.193418264389038,0.0 +109000,0.3590807020664215,1.4147617816925049,,,,,,,,,,,,,,,,, +109500,0.3222141563892364,1.3646535873413086,,,,,,,,,,,,,,,,, +110000,0.3405612409114837,1.4591789245605469,,,,,,,,,,,,,,,,, +110500,0.3372413218021393,1.4353336095809937,,,,,,,,,,,,,,,,, +111000,0.3370302021503448,1.4590778350830078,,,,,,,,,,,,,,,,, +111500,0.3281998932361603,1.4723604917526243,,,,,,,,,,,,,,,,, +112000,0.3591086566448211,1.3767088651657104,,,,,,,,,,,,,,,,, +112500,0.3439723551273346,1.4279788732528689,,,,,,,,,,,,,,,,, +112874,,,0.7050293330295608,1.311290279518141,36.558273890135176,0.6926014556546106,1.383067165937186,30.66006395991536,3000.0,0.7111730869792574,1.272040954912556,30.69741001883253,3003.0,17410.203202724457,24371.669730186462,17410.203202724457,6894.144144058228,1.232969045639038,0.0 +113000,0.3370486497879028,1.4028387069702148,,,,,,,,,,,,,,,,, +113500,0.3348321616649627,1.353950262069702,,,,,,,,,,,,,,,,, +114000,0.3479984700679779,1.4515421390533447,,,,,,,,,,,,,,,,, +114500,0.3341486155986786,1.435347080230713,,,,,,,,,,,,,,,,, +115000,0.3370992541313171,1.418031930923462,,,,,,,,,,,,,,,,, +115500,0.3482958078384399,1.3494253158569336,,,,,,,,,,,,,,,,, +116000,0.3380669057369232,1.3502166271209717,,,,,,,,,,,,,,,,, +116500,0.3361073136329651,1.376165747642517,,,,,,,,,,,,,,,,, +117000,0.3459105789661407,1.3811460733413696,,,,,,,,,,,,,,,,, +117060,,,0.7029898124061744,1.3130739653404078,36.60939641849158,0.6927006484730506,1.3831094972473992,30.56808093499612,3000.0,0.7111266050781477,1.272498420341642,30.641968225785725,3003.0,18052.37412500381,25248.056270599365,18052.37412500381,7125.89671421051,1.2717673778533936,0.0 +117500,0.3361880779266357,1.4483274221420288,,,,,,,,,,,,,,,,, +118000,0.3299523890018463,1.3407024145126345,,,,,,,,,,,,,,,,, +118500,0.3346445560455322,1.4022984504699707,,,,,,,,,,,,,,,,, +119000,0.3331246674060821,1.381550431251526,,,,,,,,,,,,,,,,, +119500,0.3268519639968872,1.3737523555755615,,,,,,,,,,,,,,,,, +120000,0.3325319290161133,1.4158120155334473,,,,,,,,,,,,,,,,, +120500,0.3272158205509186,1.3832076787948608,,,,,,,,,,,,,,,,, +121000,0.3303971588611603,1.4979450702667236,,,,,,,,,,,,,,,,, +121245,,,0.7058369118215073,1.2999912667848137,36.67263580098895,0.6928122403937955,1.3830434332804304,30.55606891050164,3000.0,0.7112311893556447,1.2723141268665388,30.67450051412709,3003.0,18694.61083889008,26121.104176282883,18694.61083889008,7354.245838403702,1.312624216079712,0.0 +121500,0.3428358435630798,1.4409452676773071,,,,,,,,,,,,,,,,, +122000,0.3600443005561828,1.4217404127120972,,,,,,,,,,,,,,,,, +122500,0.3391572535037994,1.4111255407333374,,,,,,,,,,,,,,,,, +123000,0.3358381390571594,1.3312104940414429,,,,,,,,,,,,,,,,, +123500,0.3432051539421081,1.375102162361145,,,,,,,,,,,,,,,,, +124000,0.3261123299598694,1.439716100692749,,,,,,,,,,,,,,,,, +124500,0.3321066498756408,1.338370680809021,,,,,,,,,,,,,,,,, +125000,0.3365761339664459,1.4019614458084106,,,,,,,,,,,,,,,,, +125431,,,0.7039611557092633,1.313814891981494,36.44912214972385,0.6925766574500006,1.382496613495183,30.57292623010075,3000.0,0.7109406774737087,1.271411451978386,30.791423307933886,3003.0,19336.86342072487,26990.267922639847,19336.86342072487,7578.698003530502,1.352147340774536,0.0 +125500,0.3591015040874481,1.3741400241851809,,,,,,,,,,,,,,,,, +126000,0.3132772147655487,1.3458048105239868,,,,,,,,,,,,,,,,, +126500,0.3161243498325348,1.423470973968506,,,,,,,,,,,,,,,,, +127000,0.3322039246559143,1.3097476959228516,,,,,,,,,,,,,,,,, +127500,0.3541815876960754,1.4410693645477295,,,,,,,,,,,,,,,,, +128000,0.3573786318302154,1.376853346824646,,,,,,,,,,,,,,,,, +128500,0.3187412619590759,1.378099799156189,,,,,,,,,,,,,,,,, +129000,0.3587780892848968,1.3264731168746948,,,,,,,,,,,,,,,,, +129500,0.3181833028793335,1.4289942979812622,,,,,,,,,,,,,,,,, +129613,,,0.7036231554017125,1.3080286994443433,36.59046463044769,0.6921798861762408,1.3833814056862284,30.45512799165426,3000.0,0.7110917436523154,1.272092883911452,30.75789742130872,3003.0,19979.07295823097,27860.41013264656,19979.07295823097,7804.16498541832,1.3926682472229004,0.0 +130000,0.3343880474567413,1.479011297225952,,,,,,,,,,,,,,,,, +130500,0.3570750951766968,1.3795974254608154,,,,,,,,,,,,,,,,, +131000,0.3404681384563446,1.4341920614242554,,,,,,,,,,,,,,,,, +131500,0.3262973129749298,1.337276577949524,,,,,,,,,,,,,,,,, +132000,0.3275915086269378,1.432143211364746,,,,,,,,,,,,,,,,, +132500,0.3430702686309814,1.3662161827087402,,,,,,,,,,,,,,,,, +133000,0.3351747691631317,1.3579872846603394,,,,,,,,,,,,,,,,, +133500,0.3419004082679748,1.3761169910430908,,,,,,,,,,,,,,,,, +133800,,,0.7033408232865306,1.3139929410993123,36.58855409087423,0.6931718143606402,1.3845449839431625,30.54034889300614,3000.0,0.7109174365231538,1.2735027743884726,30.66517763908644,3003.0,20621.282261371613,28737.47842621804,20621.282261371613,8036.569143533707,1.4343180656433103,0.0 +134000,0.3565320968627929,1.372496485710144,,,,,,,,,,,,,,,,, +134500,0.3334687352180481,1.35405433177948,,,,,,,,,,,,,,,,, +135000,0.3434310555458069,1.4096330404281616,,,,,,,,,,,,,,,,, +135500,0.3194648921489715,1.3650915622711182,,,,,,,,,,,,,,,,, +136000,0.3672835528850555,1.433945655822754,,,,,,,,,,,,,,,,, +136500,0.3353090584278106,1.4245092868804932,,,,,,,,,,,,,,,,, +137000,0.3261283338069916,1.42937433719635,,,,,,,,,,,,,,,,, +137500,0.3289774656295776,1.4250590801239014,,,,,,,,,,,,,,,,, +137989,,,0.7046032741999886,1.3126328895955734,36.69733703822024,0.6924526664269507,1.3848173767219254,30.68669867883425,3000.0,0.711266050781477,1.2750446662018478,30.677366900510805,3003.0,21263.457713603973,29601.30152130127,21263.457713603973,8255.743970632553,1.4763462543487549,0.0 +138000,0.3380023539066314,1.3280824422836304,,,,,,,,,,,,,,,,, +138500,0.3276697993278503,1.4177967309951782,,,,,,,,,,,,,,,,, +139000,0.3351180851459503,1.4610133171081543,,,,,,,,,,,,,,,,, +139500,0.3574661016464233,1.3892024755477903,,,,,,,,,,,,,,,,, +140000,0.356669157743454,1.3186122179031372,,,,,,,,,,,,,,,,, +140500,0.3367966711521148,1.4052236080169678,,,,,,,,,,,,,,,,, +141000,0.338977575302124,1.399154782295227,,,,,,,,,,,,,,,,, +141500,0.3476296663284302,1.497383713722229,,,,,,,,,,,,,,,,, +142000,0.3954565525054931,1.4695676565170288,,,,,,,,,,,,,,,,, +142179,,,0.7047295531748106,1.3150405869053976,36.33011062295734,0.6910639669687915,1.3910401761912436,30.552253592969947,3000.0,0.7107547498692697,1.2789763813839985,30.87643012927865,3003.0,21905.687613487244,30479.034685611725,21905.687613487244,8488.795366764069,1.5180323123931885,0.0 +142500,0.3936523199081421,1.4677045345306396,,,,,,,,,,,,,,,,, +143000,0.3894926309585571,1.3929612636566162,,,,,,,,,,,,,,,,, +143500,0.3327016532421112,1.43622088432312,,,,,,,,,,,,,,,,, +144000,0.3489832282066345,1.4263652563095093,,,,,,,,,,,,,,,,, +144500,0.3433640003204345,1.3475853204727173,,,,,,,,,,,,,,,,, +145000,0.3634448051452636,1.450786590576172,,,,,,,,,,,,,,,,, +145500,0.3465692102909088,1.412053108215332,,,,,,,,,,,,,,,,, +146000,0.3579453825950622,1.4652016162872314,,,,,,,,,,,,,,,,, +146367,,,0.707080526550545,1.295063159760742,36.38857651114236,0.6911011642757064,1.3908255167325885,30.47672666594673,3000.0,0.7103945151356691,1.2809849986926964,30.70172380829016,3003.0,22547.857746839523,31334.860281705856,22547.857746839523,8699.996084451675,1.5598969459533691,0.0 +146500,0.3712442219257355,1.399200201034546,,,,,,,,,,,,,,,,, +147000,0.3627782166004181,1.4921550750732422,,,,,,,,,,,,,,,,, +147500,0.3730242550373077,1.333629488945007,,,,,,,,,,,,,,,,, +148000,0.3601684868335724,1.431593418121338,,,,,,,,,,,,,,,,, +148500,0.3621529638767242,1.3760629892349243,,,,,,,,,,,,,,,,, +149000,0.3337656259536743,1.4371941089630127,,,,,,,,,,,,,,,,, +149500,0.3413021266460418,1.430511236190796,,,,,,,,,,,,,,,,, +150000,0.3459751904010772,1.43938148021698,,,,,,,,,,,,,,,,, +150500,0.3557848036289215,1.4669018983840942,,,,,,,,,,,,,,,,, +150557,,,0.7118764139948354,1.2636284464338765,36.85572995214668,0.6913739445264163,1.3958163491463218,30.45687353008207,3000.0,0.7088606123990471,1.289014202399628,30.537840963292496,3003.0,23190.091015338898,32189.50274205208,23190.091015338898,8909.949897766113,1.6021971702575684,0.0 +151000,0.3360735177993774,1.4796098470687866,,,,,,,,,,,,,,,,, +151500,0.3643504083156585,1.519323468208313,,,,,,,,,,,,,,,,, +152000,0.3496420979499817,1.456884503364563,,,,,,,,,,,,,,,,, +152500,0.3674136400222778,1.4418327808380127,,,,,,,,,,,,,,,,, +153000,0.3440545201301574,1.41015887260437,,,,,,,,,,,,,,,,, +153500,0.3506447374820709,1.4147071838378906,,,,,,,,,,,,,,,,, +154000,0.3529818058013916,1.4379326105117798,,,,,,,,,,,,,,,,, +154500,0.3596553206443786,1.4882031679153442,,,,,,,,,,,,,,,,, +154746,,,0.7008275420629115,1.330113397694541,35.98773863758199,0.6904688100581517,1.3992569062999838,30.40823285154981,3000.0,0.7078844924757423,1.291251053105572,30.342683619124177,3003.0,23832.328326940536,33054.1152844429,23832.328326940536,9129.864770650864,1.6439316272735596,0.0 +155000,0.3407307267189026,1.4655789136886597,,,,,,,,,,,,,,,,, +155500,0.3740216791629791,1.463107943534851,,,,,,,,,,,,,,,,, +156000,0.4023193120956421,1.4454957246780396,,,,,,,,,,,,,,,,, +156500,0.382112056016922,1.4364521503448486,,,,,,,,,,,,,,,,, +157000,0.3847205936908722,1.444832444190979,,,,,,,,,,,,,,,,, +157500,0.3515453338623047,1.4516841173171997,,,,,,,,,,,,,,,,, +158000,0.3576461970806122,1.5069721937179563,,,,,,,,,,,,,,,,, +158500,0.3798002004623413,1.3965855836868286,,,,,,,,,,,,,,,,, +158936,,,,,,,,,,,,,,24474.575791597366,,,,,0.0 diff --git a/logs/self_tuning/ademamix_golden/study_2/wmt_pytorch/trial_1/meta_data_0.json b/logs/self_tuning/ademamix_golden/study_2/wmt_pytorch/trial_1/meta_data_0.json new file mode 100644 index 00000000..6c094bc3 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/wmt_pytorch/trial_1/meta_data_0.json @@ -0,0 +1,71 @@ +{ + "workload.activation": "relu", + "workload.attention_temp": 1.0, + "workload.eval_batch_size": 128, + "workload.eval_period_time_sec": 644, + "workload.glu": false, + "workload.max_allowed_runtime_sec": 16114, + "workload.num_eval_train_examples": 3072, + "workload.num_test_examples": 3003, + "workload.num_train_examples": 5906184, + "workload.num_validation_examples": 3000, + "workload.pre_ln": true, + "workload.step_hint": 120000, + "workload.target_metric_name": "bleu", + "workload.test_target_value": 30.7219, + "workload.train_mean": 30.7219, + "workload.train_stddev": 30.7219, + "workload.validation_target_value": 30.8491, + "cpu.util.avg_percent_since_last": 4.6, + "cpu.freq.current": 2200.1959999999985, + "mem.total": 359053524992, + "mem.available": 348316020736, + "mem.used": 6717587456, + "mem.percent_used": 3.0, + "mem.read_bytes_since_boot": 48170199430656, + "mem.write_bytes_since_boot": 201409144320, + "net.bytes_sent_since_boot": 31339618, + "net.bytes_recv_since_boot": 31342711, + "gpu.count": 4, + "gpu.0.compute.util": 0.0, + "gpu.0.mem.util": 0.0647705078125, + "gpu.0.mem.total": 40960.0, + "gpu.0.mem.used": 2653.0, + "gpu.0.mem.free": 37674.0, + "gpu.0.temp.current": 32.0, + "gpu.1.compute.util": 0.0, + "gpu.1.mem.util": 0.0647705078125, + "gpu.1.mem.total": 40960.0, + "gpu.1.mem.used": 2653.0, + "gpu.1.mem.free": 37674.0, + "gpu.1.temp.current": 30.0, + "gpu.2.compute.util": 0.0, + "gpu.2.mem.util": 0.0647705078125, + "gpu.2.mem.total": 40960.0, + "gpu.2.mem.used": 2653.0, + "gpu.2.mem.free": 37674.0, + "gpu.2.temp.current": 30.0, + "gpu.3.compute.util": 0.0, + "gpu.3.mem.util": 0.0647705078125, + "gpu.3.mem.total": 40960.0, + "gpu.3.mem.used": 2653.0, + "gpu.3.mem.free": 37674.0, + "gpu.3.temp.current": 32.0, + "gpu.avg.compute.util": 0.0, + "gpu.avg.mem.util": 0.0647705078125, + "gpu.avg.mem.total": 40960.0, + "gpu.avg.mem.used": 2653.0, + "gpu.avg.mem.free": 37674.0, + "gpu.avg.temp.current": 31.0, + "os_platform": "Linux-6.1.0-44-cloud-amd64-x86_64-with-glibc2.31", + "python_version": "3.11.10", + "python_compiler": "GCC 9.4.0", + "git_branch": "main", + "git_commit_hash": "b21be29be0a1573fb4f78f849aea019cdb520862", + "cpu_model_name": "Intel(R) Xeon(R) CPU @ 2.20GHz", + "cpu_count": 24, + "gpu_model_name": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_driver": "550.90.12", + "rng_seed": 16914702 +} \ No newline at end of file diff --git a/logs/self_tuning/ademamix_golden/study_2/wmt_pytorch/wmt_pytorch_09-16-2026-15-00-16.log b/logs/self_tuning/ademamix_golden/study_2/wmt_pytorch/wmt_pytorch_09-16-2026-15-00-16.log new file mode 100644 index 00000000..4846ebe9 --- /dev/null +++ b/logs/self_tuning/ademamix_golden/study_2/wmt_pytorch/wmt_pytorch_09-16-2026-15-00-16.log @@ -0,0 +1,1272 @@ +torchrun --redirects 1:0,2:0,3:0 --standalone --nnodes=1 --nproc_per_node=4 submission_runner.py --framework=pytorch --workload=wmt --submission_path=submissions_algorithms/submissions/self_tuning/ademamix_golden/submission.py --data_dir=/data/wmt --experiment_dir=/experiment_runs --experiment_name=submissions_a100/ademamix_golden/study_2 --overwrite=True --save_checkpoints=False --rng_seed=16914702 --torch_compile=true --tuning_ruleset=self 2>&1 | tee -a /logs/wmt_pytorch_09-16-2026-15-00-16.log +W0916 15:00:41.579000 9 site-packages/torch/distributed/run.py:803] +W0916 15:00:41.579000 9 site-packages/torch/distributed/run.py:803] ***************************************** +W0916 15:00:41.579000 9 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0916 15:00:41.579000 9 site-packages/torch/distributed/run.py:803] ***************************************** +2026-09-16 15:00:56.950289: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 15:00:56.950289: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 15:00:56.950289: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-09-16 15:00:56.950301: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789570857.556658 39 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789570857.556666 38 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789570857.556671 40 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +WARNING: All log messages before absl::InitializeLog() is called are written to STDERR +E0000 00:00:1789570857.556681 41 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +E0000 00:00:1789570857.683640 40 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789570857.683649 38 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789570857.683650 41 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +E0000 00:00:1789570857.683667 39 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +W0000 00:00:1789570859.100809 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789570859.100807 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789570859.100809 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789570859.100808 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789570859.100848 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789570859.100848 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789570859.100848 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789570859.100850 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789570859.100849 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789570859.100851 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789570859.100851 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789570859.100853 38 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789570859.100853 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789570859.100853 41 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789570859.100854 40 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789570859.100855 39 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once. +W0000 00:00:1789570887.820774 38 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789570887.820777 39 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789570887.820776 41 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +W0000 00:00:1789570887.824600 40 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. +Skipping registering GPU devices... +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/torch/__init__.py:1617: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.) + _C._set_float32_matmul_precision(precision) +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/usr/local/lib/python3.11/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W916 15:01:38.609638257 ProcessGroupNCCL.cpp:5068] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +I0916 15:01:40.999319 140285881828544 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/wmt_pytorch. +I0916 15:01:40.999317 140207328986304 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/wmt_pytorch. +I0916 15:01:40.999312 140231027893440 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/wmt_pytorch. +I0916 15:01:40.999347 140350976722112 logger_utils.py:84] Creating experiment directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/wmt_pytorch. +I0916 15:01:41.207547 140285881828544 submission_runner.py:741] Creating directory at /experiment_runs/submissions_a100/ademamix_golden/study_2/wmt_pytorch/trial_1. +I0916 15:01:41.482823 140285881828544 submission_runner.py:242] Initializing dataset. +I0916 15:01:41.483000 140285881828544 submission_runner.py:251] Initializing model. +I0916 15:01:53.519309 140285881828544 submission_runner.py:290] Performing `torch.compile`. +I0916 15:01:57.429068 140285881828544 submission_runner.py:294] Initializing optimizer. +I0916 15:01:57.429999 140285881828544 submission_runner.py:299] Initializing metrics bundle. +I0916 15:01:57.430167 140285881828544 submission_runner.py:321] Initializing checkpoint and logger. +I0916 15:01:57.430990 140285881828544 submission_runner.py:344] Saving meta data to /experiment_runs/submissions_a100/ademamix_golden/study_2/wmt_pytorch/trial_1/meta_data_0.json. +I0916 15:01:57.431058 140231027893440 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 15:01:57.431069 140207328986304 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 15:01:57.431077 140350976722112 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 15:01:57.431191 140285881828544 logger_utils.py:283] Unable to record workload.train_mean information. Continuing without it. +I0916 15:01:57.431215 140207328986304 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 15:01:57.431215 140231027893440 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 15:01:57.431227 140350976722112 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 15:01:57.431248 140285881828544 logger_utils.py:283] Unable to record workload.train_stddev information. Continuing without it. +I0916 15:01:57.814631 140285881828544 submission_runner.py:348] Saving flags to /experiment_runs/submissions_a100/ademamix_golden/study_2/wmt_pytorch/trial_1/flags_0.json. +I0916 15:01:57.914602 140285881828544 submission_runner.py:359] Starting training loop. +I0916 15:01:59.078328 140285881828544 dataset_info.py:707] Load dataset info from /data/wmt/wmt17_translate/de-en/1.0.0 +I0916 15:01:59.092786 140285881828544 dataset_info.py:793] For 'wmt17_translate/de-en/1.0.0': fields info.[splits, supervised_keys] differ on disk and in the code. Keeping the one from code. +I0916 15:01:59.143158 140285881828544 reader.py:262] Creating a tf.data.Dataset reading 16 files located in folders: /data/wmt/wmt17_translate/de-en/1.0.0. +I0916 15:01:59.275431 140285881828544 logging_logger.py:49] Constructing tf.data.Dataset wmt17_translate for split train, from /data/wmt/wmt17_translate/de-en/1.0.0 +[rank2]:W0916 15:02:01.578000 40 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank3]:W0916 15:02:01.578000 41 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank1]:W0916 15:02:01.578000 39 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +[rank0]:W0916 15:02:02.499000 38 site-packages/torch/_logging/_internal.py:1199] [0/0] Profiler function will be ignored +I0916 15:03:09.173643 140261057283840 logging_writer.py:48] [0] global_step=0, grad_norm=0.5, loss=11.5939 +I0916 15:03:09.429928 140285881828544 submission.py:307] 0) loss = 11.594, grad_norm = 0.500 +I0916 15:03:10.122759 140285881828544 spec.py:333] Evaluating on the training split. +I0916 15:03:10.124703 140285881828544 dataset_info.py:707] Load dataset info from /data/wmt/wmt17_translate/de-en/1.0.0 +I0916 15:03:10.126821 140285881828544 dataset_info.py:793] For 'wmt17_translate/de-en/1.0.0': fields info.[splits, supervised_keys] differ on disk and in the code. Keeping the one from code. +I0916 15:03:10.127770 140285881828544 reader.py:262] Creating a tf.data.Dataset reading 16 files located in folders: /data/wmt/wmt17_translate/de-en/1.0.0. +I0916 15:03:10.161703 140285881828544 logging_logger.py:49] Constructing tf.data.Dataset wmt17_translate for split train, from /data/wmt/wmt17_translate/de-en/1.0.0 +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +/usr/local/lib/python3.11/site-packages/torch/_dynamo/variables/functions.py:1598: UserWarning: Dynamo does not know how to trace the builtin `torch._C._distributed_c10d.pybind11_detail_function_record_v1_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1._broadcast_coalesced.` This function is either a Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python extension (perhaps created with pybind). +If it is a Python builtin, please file an issue on GitHub so the PyTorch team can add support for it and see the next case for a workaround. +If it is a third-party C/C++ Python extension, please either wrap it into a PyTorch-understood custom operator (see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html for more details) or, if it is traceable, use `torch.compiler.allow_in_graph`. + torch._dynamo.utils.warn_once(explanation + "\n" + "\n".join(hints)) +I0916 15:03:45.153820 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 15:06:00.201044 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 15:06:00.267346 140285881828544 dataset_info.py:707] Load dataset info from /data/wmt/wmt14_translate/de-en/1.0.0 +I0916 15:06:00.278856 140285881828544 dataset_info.py:793] For 'wmt14_translate/de-en/1.0.0': fields info.[splits, supervised_keys] differ on disk and in the code. Keeping the one from code. +I0916 15:06:00.279597 140285881828544 reader.py:262] Creating a tf.data.Dataset reading 1 files located in folders: /data/wmt/wmt14_translate/de-en/1.0.0. +I0916 15:06:00.314409 140285881828544 logging_logger.py:49] Constructing tf.data.Dataset wmt14_translate for split validation, from /data/wmt/wmt14_translate/de-en/1.0.0 +I0916 15:06:02.679830 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 15:08:14.609303 140285881828544 spec.py:363] Evaluating on the test split. +I0916 15:08:14.611346 140285881828544 dataset_info.py:707] Load dataset info from /data/wmt/wmt14_translate/de-en/1.0.0 +I0916 15:08:14.613552 140285881828544 dataset_info.py:793] For 'wmt14_translate/de-en/1.0.0': fields info.[splits, supervised_keys] differ on disk and in the code. Keeping the one from code. +I0916 15:08:14.614323 140285881828544 reader.py:262] Creating a tf.data.Dataset reading 1 files located in folders: /data/wmt/wmt14_translate/de-en/1.0.0. +I0916 15:08:14.648947 140285881828544 logging_logger.py:49] Constructing tf.data.Dataset wmt14_translate for split test, from /data/wmt/wmt14_translate/de-en/1.0.0 +I0916 15:08:16.969231 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 15:10:29.023514 140285881828544 submission_runner.py:516] Time since start: 511.11s, Step: 1, {'train/accuracy': 0.0005829504149235306, 'train/loss': 11.773135844592277, 'train/bleu': 0.0, 'validation/accuracy': 0.00048356498989473163, 'validation/loss': 11.785403311800225, 'validation/bleu': 0.0, 'validation/num_examples': 3000, 'test/accuracy': 0.0007088489919237697, 'test/loss': 11.775305037476032, 'test/bleu': 0.0, 'test/num_examples': 3003, 'score': 71.5164270401001, 'total_duration': 511.1090278625488, 'accumulated_submission_time': 71.5164270401001, 'accumulated_eval_time': 438.90079641342163, 'accumulated_logging_time': 0} +I0916 15:10:29.127816 140257433470720 logging_writer.py:48] [1] accumulated_eval_time=438.901, accumulated_logging_time=0, accumulated_submission_time=71.5164, global_step=1, preemption_count=0, score=71.5164, test/accuracy=0.000708849, test/bleu=0, test/loss=11.7753, test/num_examples=3003, total_duration=511.109, train/accuracy=0.00058295, train/bleu=0, train/loss=11.7731, validation/accuracy=0.000483565, validation/bleu=0, validation/loss=11.7854, validation/num_examples=3000 +I0916 15:10:30.068186 140257425078016 logging_writer.py:48] [1] global_step=1, grad_norm=0.5, loss=11.5941 +I0916 15:10:30.071173 140285881828544 submission.py:307] 1) loss = 11.594, grad_norm = 0.500 +I0916 15:10:30.238852 140257433470720 logging_writer.py:48] [2] global_step=2, grad_norm=0.5, loss=11.5968 +I0916 15:10:30.241794 140285881828544 submission.py:307] 2) loss = 11.597, grad_norm = 0.500 +I0916 15:10:30.409993 140257425078016 logging_writer.py:48] [3] global_step=3, grad_norm=0.5, loss=11.5788 +I0916 15:10:30.412944 140285881828544 submission.py:307] 3) loss = 11.579, grad_norm = 0.500 +I0916 15:10:30.580789 140257433470720 logging_writer.py:48] [4] global_step=4, grad_norm=0.5, loss=11.5646 +I0916 15:10:30.583653 140285881828544 submission.py:307] 4) loss = 11.565, grad_norm = 0.500 +I0916 15:10:30.752214 140257425078016 logging_writer.py:48] [5] global_step=5, grad_norm=0.5, loss=11.5428 +I0916 15:10:30.755131 140285881828544 submission.py:307] 5) loss = 11.543, grad_norm = 0.500 +I0916 15:10:30.922281 140257433470720 logging_writer.py:48] [6] global_step=6, grad_norm=0.5, loss=11.516 +I0916 15:10:30.925075 140285881828544 submission.py:307] 6) loss = 11.516, grad_norm = 0.500 +I0916 15:10:31.093202 140257425078016 logging_writer.py:48] [7] global_step=7, grad_norm=0.5, loss=11.4902 +I0916 15:10:31.096037 140285881828544 submission.py:307] 7) loss = 11.490, grad_norm = 0.500 +I0916 15:10:31.263628 140257433470720 logging_writer.py:48] [8] global_step=8, grad_norm=0.5, loss=11.4461 +I0916 15:10:31.266582 140285881828544 submission.py:307] 8) loss = 11.446, grad_norm = 0.500 +I0916 15:10:31.434804 140257425078016 logging_writer.py:48] [9] global_step=9, grad_norm=0.5, loss=11.411 +I0916 15:10:31.437665 140285881828544 submission.py:307] 9) loss = 11.411, grad_norm = 0.500 +I0916 15:10:31.604633 140257433470720 logging_writer.py:48] [10] global_step=10, grad_norm=0.5, loss=11.3508 +I0916 15:10:31.607429 140285881828544 submission.py:307] 10) loss = 11.351, grad_norm = 0.500 +I0916 15:10:31.774466 140257425078016 logging_writer.py:48] [11] global_step=11, grad_norm=0.5, loss=11.3121 +I0916 15:10:31.777401 140285881828544 submission.py:307] 11) loss = 11.312, grad_norm = 0.500 +I0916 15:10:31.945262 140257433470720 logging_writer.py:48] [12] global_step=12, grad_norm=0.5, loss=11.2577 +I0916 15:10:31.948084 140285881828544 submission.py:307] 12) loss = 11.258, grad_norm = 0.500 +I0916 15:10:32.115283 140257425078016 logging_writer.py:48] [13] global_step=13, grad_norm=0.5, loss=11.2136 +I0916 15:10:32.118105 140285881828544 submission.py:307] 13) loss = 11.214, grad_norm = 0.500 +I0916 15:10:32.286389 140257433470720 logging_writer.py:48] [14] global_step=14, grad_norm=0.5, loss=11.1485 +I0916 15:10:32.289234 140285881828544 submission.py:307] 14) loss = 11.148, grad_norm = 0.500 +I0916 15:10:32.457067 140257425078016 logging_writer.py:48] [15] global_step=15, grad_norm=0.5, loss=11.0828 +I0916 15:10:32.459907 140285881828544 submission.py:307] 15) loss = 11.083, grad_norm = 0.500 +I0916 15:10:32.627943 140257433470720 logging_writer.py:48] [16] global_step=16, grad_norm=0.5, loss=11.0311 +I0916 15:10:32.630740 140285881828544 submission.py:307] 16) loss = 11.031, grad_norm = 0.500 +I0916 15:10:32.797679 140257425078016 logging_writer.py:48] [17] global_step=17, grad_norm=0.5, loss=10.9746 +I0916 15:10:32.800479 140285881828544 submission.py:307] 17) loss = 10.975, grad_norm = 0.500 +I0916 15:10:32.968498 140257433470720 logging_writer.py:48] [18] global_step=18, grad_norm=0.5, loss=10.8991 +I0916 15:10:32.971290 140285881828544 submission.py:307] 18) loss = 10.899, grad_norm = 0.500 +I0916 15:10:33.137984 140257425078016 logging_writer.py:48] [19] global_step=19, grad_norm=0.5, loss=10.8433 +I0916 15:10:33.140821 140285881828544 submission.py:307] 19) loss = 10.843, grad_norm = 0.500 +I0916 15:10:33.308630 140257433470720 logging_writer.py:48] [20] global_step=20, grad_norm=0.5, loss=10.7532 +I0916 15:10:33.311441 140285881828544 submission.py:307] 20) loss = 10.753, grad_norm = 0.500 +I0916 15:10:33.479331 140257425078016 logging_writer.py:48] [21] global_step=21, grad_norm=0.5, loss=10.6865 +I0916 15:10:33.482189 140285881828544 submission.py:307] 21) loss = 10.686, grad_norm = 0.500 +I0916 15:10:33.650288 140257433470720 logging_writer.py:48] [22] global_step=22, grad_norm=0.5, loss=10.6419 +I0916 15:10:33.653122 140285881828544 submission.py:307] 22) loss = 10.642, grad_norm = 0.500 +I0916 15:10:33.821334 140257425078016 logging_writer.py:48] [23] global_step=23, grad_norm=0.5, loss=10.5625 +I0916 15:10:33.824111 140285881828544 submission.py:307] 23) loss = 10.563, grad_norm = 0.500 +I0916 15:10:33.991466 140257433470720 logging_writer.py:48] [24] global_step=24, grad_norm=0.5, loss=10.4856 +I0916 15:10:33.994274 140285881828544 submission.py:307] 24) loss = 10.486, grad_norm = 0.500 +I0916 15:10:34.162826 140257425078016 logging_writer.py:48] [25] global_step=25, grad_norm=0.5, loss=10.4157 +I0916 15:10:34.165708 140285881828544 submission.py:307] 25) loss = 10.416, grad_norm = 0.500 +I0916 15:10:34.333654 140257433470720 logging_writer.py:48] [26] global_step=26, grad_norm=0.5, loss=10.3445 +I0916 15:10:34.336504 140285881828544 submission.py:307] 26) loss = 10.345, grad_norm = 0.500 +I0916 15:10:34.505142 140257425078016 logging_writer.py:48] [27] global_step=27, grad_norm=0.5, loss=10.2825 +I0916 15:10:34.507948 140285881828544 submission.py:307] 27) loss = 10.282, grad_norm = 0.500 +I0916 15:10:34.675136 140257433470720 logging_writer.py:48] [28] global_step=28, grad_norm=0.5, loss=10.2067 +I0916 15:10:34.677973 140285881828544 submission.py:307] 28) loss = 10.207, grad_norm = 0.500 +I0916 15:10:34.846042 140257425078016 logging_writer.py:48] [29] global_step=29, grad_norm=0.5, loss=10.1431 +I0916 15:10:34.848926 140285881828544 submission.py:307] 29) loss = 10.143, grad_norm = 0.500 +I0916 15:10:35.015850 140257433470720 logging_writer.py:48] [30] global_step=30, grad_norm=0.5, loss=10.0695 +I0916 15:10:35.018738 140285881828544 submission.py:307] 30) loss = 10.069, grad_norm = 0.500 +I0916 15:10:35.185958 140257425078016 logging_writer.py:48] [31] global_step=31, grad_norm=0.5, loss=10.0035 +I0916 15:10:35.188885 140285881828544 submission.py:307] 31) loss = 10.004, grad_norm = 0.500 +I0916 15:10:35.356632 140257433470720 logging_writer.py:48] [32] global_step=32, grad_norm=0.5, loss=9.90752 +I0916 15:10:35.359453 140285881828544 submission.py:307] 32) loss = 9.908, grad_norm = 0.500 +I0916 15:10:35.527290 140257425078016 logging_writer.py:48] [33] global_step=33, grad_norm=0.5, loss=9.85178 +I0916 15:10:35.530181 140285881828544 submission.py:307] 33) loss = 9.852, grad_norm = 0.500 +I0916 15:10:35.698217 140257433470720 logging_writer.py:48] [34] global_step=34, grad_norm=0.5, loss=9.80323 +I0916 15:10:35.701063 140285881828544 submission.py:307] 34) loss = 9.803, grad_norm = 0.500 +I0916 15:10:35.868502 140257425078016 logging_writer.py:48] [35] global_step=35, grad_norm=0.5, loss=9.72533 +I0916 15:10:35.871357 140285881828544 submission.py:307] 35) loss = 9.725, grad_norm = 0.500 +I0916 15:10:36.039646 140257433470720 logging_writer.py:48] [36] global_step=36, grad_norm=0.5, loss=9.68883 +I0916 15:10:36.042451 140285881828544 submission.py:307] 36) loss = 9.689, grad_norm = 0.500 +I0916 15:10:36.209806 140257425078016 logging_writer.py:48] [37] global_step=37, grad_norm=0.5, loss=9.61601 +I0916 15:10:36.212646 140285881828544 submission.py:307] 37) loss = 9.616, grad_norm = 0.500 +I0916 15:10:36.380820 140257433470720 logging_writer.py:48] [38] global_step=38, grad_norm=0.5, loss=9.57462 +I0916 15:10:36.383746 140285881828544 submission.py:307] 38) loss = 9.575, grad_norm = 0.500 +I0916 15:10:36.550532 140257425078016 logging_writer.py:48] [39] global_step=39, grad_norm=0.5, loss=9.52717 +I0916 15:10:36.553474 140285881828544 submission.py:307] 39) loss = 9.527, grad_norm = 0.500 +I0916 15:10:36.721047 140257433470720 logging_writer.py:48] [40] global_step=40, grad_norm=0.5, loss=9.45085 +I0916 15:10:36.723850 140285881828544 submission.py:307] 40) loss = 9.451, grad_norm = 0.500 +I0916 15:10:36.891085 140257425078016 logging_writer.py:48] [41] global_step=41, grad_norm=0.5, loss=9.40571 +I0916 15:10:36.893933 140285881828544 submission.py:307] 41) loss = 9.406, grad_norm = 0.500 +I0916 15:10:37.061557 140257433470720 logging_writer.py:48] [42] global_step=42, grad_norm=0.5, loss=9.35555 +I0916 15:10:37.064404 140285881828544 submission.py:307] 42) loss = 9.356, grad_norm = 0.500 +I0916 15:10:37.232544 140257425078016 logging_writer.py:48] [43] global_step=43, grad_norm=0.5, loss=9.27389 +I0916 15:10:37.235368 140285881828544 submission.py:307] 43) loss = 9.274, grad_norm = 0.500 +I0916 15:10:37.402940 140257433470720 logging_writer.py:48] [44] global_step=44, grad_norm=0.5, loss=9.24452 +I0916 15:10:37.405829 140285881828544 submission.py:307] 44) loss = 9.245, grad_norm = 0.500 +I0916 15:10:37.574260 140257425078016 logging_writer.py:48] [45] global_step=45, grad_norm=0.5, loss=9.214 +I0916 15:10:37.577102 140285881828544 submission.py:307] 45) loss = 9.214, grad_norm = 0.500 +I0916 15:10:37.744369 140257433470720 logging_writer.py:48] [46] global_step=46, grad_norm=0.5, loss=9.13447 +I0916 15:10:37.747327 140285881828544 submission.py:307] 46) loss = 9.134, grad_norm = 0.500 +I0916 15:10:37.915446 140257425078016 logging_writer.py:48] [47] global_step=47, grad_norm=0.499999, loss=9.11849 +I0916 15:10:37.918227 140285881828544 submission.py:307] 47) loss = 9.118, grad_norm = 0.500 +I0916 15:10:38.085597 140257433470720 logging_writer.py:48] [48] global_step=48, grad_norm=0.499999, loss=9.0622 +I0916 15:10:38.088515 140285881828544 submission.py:307] 48) loss = 9.062, grad_norm = 0.500 +I0916 15:10:38.257049 140257425078016 logging_writer.py:48] [49] global_step=49, grad_norm=0.499999, loss=9.04249 +I0916 15:10:38.259925 140285881828544 submission.py:307] 49) loss = 9.042, grad_norm = 0.500 +I0916 15:10:38.427176 140257433470720 logging_writer.py:48] [50] global_step=50, grad_norm=0.499999, loss=8.98587 +I0916 15:10:38.430006 140285881828544 submission.py:307] 50) loss = 8.986, grad_norm = 0.500 +I0916 15:10:38.598170 140257425078016 logging_writer.py:48] [51] global_step=51, grad_norm=0.499999, loss=8.95369 +I0916 15:10:38.600955 140285881828544 submission.py:307] 51) loss = 8.954, grad_norm = 0.500 +I0916 15:10:38.768549 140257433470720 logging_writer.py:48] [52] global_step=52, grad_norm=0.499999, loss=8.91614 +I0916 15:10:38.771418 140285881828544 submission.py:307] 52) loss = 8.916, grad_norm = 0.500 +I0916 15:10:38.939858 140257425078016 logging_writer.py:48] [53] global_step=53, grad_norm=0.499999, loss=8.91605 +I0916 15:10:38.942719 140285881828544 submission.py:307] 53) loss = 8.916, grad_norm = 0.500 +I0916 15:10:39.110922 140257433470720 logging_writer.py:48] [54] global_step=54, grad_norm=0.499999, loss=8.89709 +I0916 15:10:39.113837 140285881828544 submission.py:307] 54) loss = 8.897, grad_norm = 0.500 +I0916 15:10:39.281956 140257425078016 logging_writer.py:48] [55] global_step=55, grad_norm=0.499999, loss=8.84455 +I0916 15:10:39.284839 140285881828544 submission.py:307] 55) loss = 8.845, grad_norm = 0.500 +I0916 15:10:39.454012 140257433470720 logging_writer.py:48] [56] global_step=56, grad_norm=0.499999, loss=8.82957 +I0916 15:10:39.456955 140285881828544 submission.py:307] 56) loss = 8.830, grad_norm = 0.500 +I0916 15:10:39.625035 140257425078016 logging_writer.py:48] [57] global_step=57, grad_norm=0.499999, loss=8.81918 +I0916 15:10:39.627944 140285881828544 submission.py:307] 57) loss = 8.819, grad_norm = 0.500 +I0916 15:10:39.795881 140257433470720 logging_writer.py:48] [58] global_step=58, grad_norm=0.497636, loss=8.77853 +I0916 15:10:39.798762 140285881828544 submission.py:307] 58) loss = 8.779, grad_norm = 0.498 +I0916 15:10:39.967244 140257425078016 logging_writer.py:48] [59] global_step=59, grad_norm=0.469046, loss=8.77976 +I0916 15:10:39.970185 140285881828544 submission.py:307] 59) loss = 8.780, grad_norm = 0.469 +I0916 15:10:40.138662 140257433470720 logging_writer.py:48] [60] global_step=60, grad_norm=0.448812, loss=8.70622 +I0916 15:10:40.141470 140285881828544 submission.py:307] 60) loss = 8.706, grad_norm = 0.449 +I0916 15:10:40.309139 140257425078016 logging_writer.py:48] [61] global_step=61, grad_norm=0.416589, loss=8.71571 +I0916 15:10:40.311982 140285881828544 submission.py:307] 61) loss = 8.716, grad_norm = 0.417 +I0916 15:10:40.480749 140257433470720 logging_writer.py:48] [62] global_step=62, grad_norm=0.391518, loss=8.71724 +I0916 15:10:40.483689 140285881828544 submission.py:307] 62) loss = 8.717, grad_norm = 0.392 +I0916 15:10:40.651673 140257425078016 logging_writer.py:48] [63] global_step=63, grad_norm=0.392704, loss=8.64874 +I0916 15:10:40.654563 140285881828544 submission.py:307] 63) loss = 8.649, grad_norm = 0.393 +I0916 15:10:40.822716 140257433470720 logging_writer.py:48] [64] global_step=64, grad_norm=0.370353, loss=8.66727 +I0916 15:10:40.825567 140285881828544 submission.py:307] 64) loss = 8.667, grad_norm = 0.370 +I0916 15:10:40.992603 140257425078016 logging_writer.py:48] [65] global_step=65, grad_norm=0.347388, loss=8.61915 +I0916 15:10:40.995486 140285881828544 submission.py:307] 65) loss = 8.619, grad_norm = 0.347 +I0916 15:10:41.163889 140257433470720 logging_writer.py:48] [66] global_step=66, grad_norm=0.32779, loss=8.58522 +I0916 15:10:41.166837 140285881828544 submission.py:307] 66) loss = 8.585, grad_norm = 0.328 +I0916 15:10:41.333722 140257425078016 logging_writer.py:48] [67] global_step=67, grad_norm=0.317359, loss=8.59886 +I0916 15:10:41.336633 140285881828544 submission.py:307] 67) loss = 8.599, grad_norm = 0.317 +I0916 15:10:41.504228 140257433470720 logging_writer.py:48] [68] global_step=68, grad_norm=0.304919, loss=8.603 +I0916 15:10:41.507026 140285881828544 submission.py:307] 68) loss = 8.603, grad_norm = 0.305 +I0916 15:10:41.674485 140257425078016 logging_writer.py:48] [69] global_step=69, grad_norm=0.290331, loss=8.59051 +I0916 15:10:41.677359 140285881828544 submission.py:307] 69) loss = 8.591, grad_norm = 0.290 +I0916 15:10:41.845170 140257433470720 logging_writer.py:48] [70] global_step=70, grad_norm=0.292609, loss=8.53959 +I0916 15:10:41.847998 140285881828544 submission.py:307] 70) loss = 8.540, grad_norm = 0.293 +I0916 15:10:42.015979 140257425078016 logging_writer.py:48] [71] global_step=71, grad_norm=0.28575, loss=8.54946 +I0916 15:10:42.018936 140285881828544 submission.py:307] 71) loss = 8.549, grad_norm = 0.286 +I0916 15:10:42.186966 140257433470720 logging_writer.py:48] [72] global_step=72, grad_norm=0.263323, loss=8.56694 +I0916 15:10:42.189871 140285881828544 submission.py:307] 72) loss = 8.567, grad_norm = 0.263 +I0916 15:10:42.358195 140257425078016 logging_writer.py:48] [73] global_step=73, grad_norm=0.270276, loss=8.50512 +I0916 15:10:42.361075 140285881828544 submission.py:307] 73) loss = 8.505, grad_norm = 0.270 +I0916 15:10:42.528443 140257433470720 logging_writer.py:48] [74] global_step=74, grad_norm=0.270357, loss=8.47364 +I0916 15:10:42.531338 140285881828544 submission.py:307] 74) loss = 8.474, grad_norm = 0.270 +I0916 15:10:42.699429 140257425078016 logging_writer.py:48] [75] global_step=75, grad_norm=0.280364, loss=8.53687 +I0916 15:10:42.702310 140285881828544 submission.py:307] 75) loss = 8.537, grad_norm = 0.280 +I0916 15:10:42.869543 140257433470720 logging_writer.py:48] [76] global_step=76, grad_norm=0.246947, loss=8.51999 +I0916 15:10:42.872371 140285881828544 submission.py:307] 76) loss = 8.520, grad_norm = 0.247 +I0916 15:10:43.040766 140257425078016 logging_writer.py:48] [77] global_step=77, grad_norm=0.24641, loss=8.49116 +I0916 15:10:43.043655 140285881828544 submission.py:307] 77) loss = 8.491, grad_norm = 0.246 +I0916 15:10:43.210949 140257433470720 logging_writer.py:48] [78] global_step=78, grad_norm=0.298897, loss=8.44927 +I0916 15:10:43.213813 140285881828544 submission.py:307] 78) loss = 8.449, grad_norm = 0.299 +I0916 15:10:43.382064 140257425078016 logging_writer.py:48] [79] global_step=79, grad_norm=0.228015, loss=8.49218 +I0916 15:10:43.384960 140285881828544 submission.py:307] 79) loss = 8.492, grad_norm = 0.228 +I0916 15:10:43.551992 140257433470720 logging_writer.py:48] [80] global_step=80, grad_norm=0.236166, loss=8.49582 +I0916 15:10:43.554852 140285881828544 submission.py:307] 80) loss = 8.496, grad_norm = 0.236 +I0916 15:10:43.722695 140257425078016 logging_writer.py:48] [81] global_step=81, grad_norm=0.227626, loss=8.50066 +I0916 15:10:43.725571 140285881828544 submission.py:307] 81) loss = 8.501, grad_norm = 0.228 +I0916 15:10:43.893311 140257433470720 logging_writer.py:48] [82] global_step=82, grad_norm=0.242221, loss=8.3859 +I0916 15:10:43.896171 140285881828544 submission.py:307] 82) loss = 8.386, grad_norm = 0.242 +I0916 15:10:44.064199 140257425078016 logging_writer.py:48] [83] global_step=83, grad_norm=0.222375, loss=8.47048 +I0916 15:10:44.067012 140285881828544 submission.py:307] 83) loss = 8.470, grad_norm = 0.222 +I0916 15:10:44.235451 140257433470720 logging_writer.py:48] [84] global_step=84, grad_norm=0.213189, loss=8.43769 +I0916 15:10:44.238337 140285881828544 submission.py:307] 84) loss = 8.438, grad_norm = 0.213 +I0916 15:10:44.406343 140257425078016 logging_writer.py:48] [85] global_step=85, grad_norm=0.243883, loss=8.45345 +I0916 15:10:44.409301 140285881828544 submission.py:307] 85) loss = 8.453, grad_norm = 0.244 +I0916 15:10:44.577501 140257433470720 logging_writer.py:48] [86] global_step=86, grad_norm=0.224452, loss=8.42695 +I0916 15:10:44.580280 140285881828544 submission.py:307] 86) loss = 8.427, grad_norm = 0.224 +I0916 15:10:44.748314 140257425078016 logging_writer.py:48] [87] global_step=87, grad_norm=0.210763, loss=8.4091 +I0916 15:10:44.751148 140285881828544 submission.py:307] 87) loss = 8.409, grad_norm = 0.211 +I0916 15:10:44.919638 140257433470720 logging_writer.py:48] [88] global_step=88, grad_norm=0.218384, loss=8.39457 +I0916 15:10:44.922584 140285881828544 submission.py:307] 88) loss = 8.395, grad_norm = 0.218 +I0916 15:10:45.090394 140257425078016 logging_writer.py:48] [89] global_step=89, grad_norm=0.218606, loss=8.40249 +I0916 15:10:45.093352 140285881828544 submission.py:307] 89) loss = 8.402, grad_norm = 0.219 +I0916 15:10:45.262546 140257433470720 logging_writer.py:48] [90] global_step=90, grad_norm=0.228015, loss=8.39957 +I0916 15:10:45.265419 140285881828544 submission.py:307] 90) loss = 8.400, grad_norm = 0.228 +I0916 15:10:45.433134 140257425078016 logging_writer.py:48] [91] global_step=91, grad_norm=0.212608, loss=8.40092 +I0916 15:10:45.436054 140285881828544 submission.py:307] 91) loss = 8.401, grad_norm = 0.213 +I0916 15:10:45.604701 140257433470720 logging_writer.py:48] [92] global_step=92, grad_norm=0.210382, loss=8.39185 +I0916 15:10:45.607555 140285881828544 submission.py:307] 92) loss = 8.392, grad_norm = 0.210 +I0916 15:10:45.774746 140257425078016 logging_writer.py:48] [93] global_step=93, grad_norm=0.206785, loss=8.37579 +I0916 15:10:45.777639 140285881828544 submission.py:307] 93) loss = 8.376, grad_norm = 0.207 +I0916 15:10:45.945814 140257433470720 logging_writer.py:48] [94] global_step=94, grad_norm=0.212989, loss=8.37607 +I0916 15:10:45.948648 140285881828544 submission.py:307] 94) loss = 8.376, grad_norm = 0.213 +I0916 15:10:46.116064 140257425078016 logging_writer.py:48] [95] global_step=95, grad_norm=0.198815, loss=8.40665 +I0916 15:10:46.118903 140285881828544 submission.py:307] 95) loss = 8.407, grad_norm = 0.199 +I0916 15:10:46.287311 140257433470720 logging_writer.py:48] [96] global_step=96, grad_norm=0.2215, loss=8.32967 +I0916 15:10:46.290335 140285881828544 submission.py:307] 96) loss = 8.330, grad_norm = 0.222 +I0916 15:10:46.458181 140257425078016 logging_writer.py:48] [97] global_step=97, grad_norm=0.21146, loss=8.33888 +I0916 15:10:46.460979 140285881828544 submission.py:307] 97) loss = 8.339, grad_norm = 0.211 +I0916 15:10:46.629240 140257433470720 logging_writer.py:48] [98] global_step=98, grad_norm=0.221392, loss=8.3373 +I0916 15:10:46.632018 140285881828544 submission.py:307] 98) loss = 8.337, grad_norm = 0.221 +I0916 15:10:46.799721 140257425078016 logging_writer.py:48] [99] global_step=99, grad_norm=0.197932, loss=8.36178 +I0916 15:10:46.802555 140285881828544 submission.py:307] 99) loss = 8.362, grad_norm = 0.198 +I0916 15:10:46.970738 140257433470720 logging_writer.py:48] [100] global_step=100, grad_norm=0.20229, loss=8.3283 +I0916 15:10:46.973613 140285881828544 submission.py:307] 100) loss = 8.328, grad_norm = 0.202 +I0916 15:11:48.443278 140257425078016 logging_writer.py:48] [500] global_step=500, grad_norm=0.499999, loss=6.69173 +I0916 15:11:48.446364 140285881828544 submission.py:307] 500) loss = 6.692, grad_norm = 0.500 +I0916 15:13:05.398509 140257433470720 logging_writer.py:48] [1000] global_step=1000, grad_norm=0.499999, loss=5.54753 +I0916 15:13:05.401789 140285881828544 submission.py:307] 1000) loss = 5.548, grad_norm = 0.500 +I0916 15:14:22.417148 140257425078016 logging_writer.py:48] [1500] global_step=1500, grad_norm=0.499999, loss=4.68119 +I0916 15:14:22.420265 140285881828544 submission.py:307] 1500) loss = 4.681, grad_norm = 0.500 +I0916 15:15:39.452020 140257433470720 logging_writer.py:48] [2000] global_step=2000, grad_norm=0.499999, loss=3.9244 +I0916 15:15:39.455152 140285881828544 submission.py:307] 2000) loss = 3.924, grad_norm = 0.500 +I0916 15:16:56.482689 140257425078016 logging_writer.py:48] [2500] global_step=2500, grad_norm=0.499999, loss=3.39872 +I0916 15:16:56.485767 140285881828544 submission.py:307] 2500) loss = 3.399, grad_norm = 0.500 +I0916 15:18:13.562728 140257433470720 logging_writer.py:48] [3000] global_step=3000, grad_norm=0.438612, loss=3.12248 +I0916 15:18:13.565958 140285881828544 submission.py:307] 3000) loss = 3.122, grad_norm = 0.439 +I0916 15:19:30.615511 140257425078016 logging_writer.py:48] [3500] global_step=3500, grad_norm=0.425122, loss=2.88062 +I0916 15:19:30.618682 140285881828544 submission.py:307] 3500) loss = 2.881, grad_norm = 0.425 +I0916 15:20:47.708444 140257433470720 logging_writer.py:48] [4000] global_step=4000, grad_norm=0.393195, loss=2.83296 +I0916 15:20:47.711526 140285881828544 submission.py:307] 4000) loss = 2.833, grad_norm = 0.393 +I0916 15:21:13.727835 140285881828544 spec.py:333] Evaluating on the training split. +I0916 15:21:15.927493 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 15:22:20.385892 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 15:22:22.565060 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 15:23:26.438003 140285881828544 spec.py:363] Evaluating on the test split. +I0916 15:23:28.623354 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 15:24:30.220721 140285881828544 submission_runner.py:516] Time since start: 1352.31s, Step: 4166, {'train/accuracy': 0.5465434516243545, 'train/loss': 2.6136053282489953, 'train/bleu': 25.524355173817682, 'validation/accuracy': 0.5516484606514488, 'validation/loss': 2.552473388426678, 'validation/bleu': 21.09654901711181, 'validation/num_examples': 3000, 'test/accuracy': 0.551833129975016, 'test/loss': 2.5637898727557955, 'test/bleu': 19.5996366388383, 'test/num_examples': 3003, 'score': 713.5417437553406, 'total_duration': 1352.306256055832, 'accumulated_submission_time': 713.5417437553406, 'accumulated_eval_time': 635.3937435150146, 'accumulated_logging_time': 0.11404705047607422} +I0916 15:24:30.365928 140257425078016 logging_writer.py:48] [4166] accumulated_eval_time=635.394, accumulated_logging_time=0.114047, accumulated_submission_time=713.542, global_step=4166, preemption_count=0, score=713.542, test/accuracy=0.551833, test/bleu=19.5996, test/loss=2.56379, test/num_examples=3003, total_duration=1352.31, train/accuracy=0.546543, train/bleu=25.5244, train/loss=2.61361, validation/accuracy=0.551648, validation/bleu=21.0965, validation/loss=2.55247, validation/num_examples=3000 +I0916 15:25:22.440389 140257433470720 logging_writer.py:48] [4500] global_step=4500, grad_norm=0.375981, loss=2.68692 +I0916 15:25:22.443682 140285881828544 submission.py:307] 4500) loss = 2.687, grad_norm = 0.376 +I0916 15:26:39.348602 140257425078016 logging_writer.py:48] [5000] global_step=5000, grad_norm=0.331693, loss=2.48134 +I0916 15:26:39.351764 140285881828544 submission.py:307] 5000) loss = 2.481, grad_norm = 0.332 +I0916 15:27:56.363597 140257433470720 logging_writer.py:48] [5500] global_step=5500, grad_norm=0.328729, loss=2.53667 +I0916 15:27:56.366688 140285881828544 submission.py:307] 5500) loss = 2.537, grad_norm = 0.329 +I0916 15:29:13.341030 140257425078016 logging_writer.py:48] [6000] global_step=6000, grad_norm=0.296807, loss=2.3312 +I0916 15:29:13.344174 140285881828544 submission.py:307] 6000) loss = 2.331, grad_norm = 0.297 +I0916 15:30:30.322952 140257433470720 logging_writer.py:48] [6500] global_step=6500, grad_norm=0.294558, loss=2.38358 +I0916 15:30:30.326392 140285881828544 submission.py:307] 6500) loss = 2.384, grad_norm = 0.295 +I0916 15:31:47.274308 140257425078016 logging_writer.py:48] [7000] global_step=7000, grad_norm=0.294637, loss=2.53232 +I0916 15:31:47.277391 140285881828544 submission.py:307] 7000) loss = 2.532, grad_norm = 0.295 +I0916 15:33:04.247955 140257433470720 logging_writer.py:48] [7500] global_step=7500, grad_norm=0.273706, loss=2.27717 +I0916 15:33:04.251235 140285881828544 submission.py:307] 7500) loss = 2.277, grad_norm = 0.274 +I0916 15:34:21.214821 140257425078016 logging_writer.py:48] [8000] global_step=8000, grad_norm=0.248093, loss=2.25815 +I0916 15:34:21.217848 140285881828544 submission.py:307] 8000) loss = 2.258, grad_norm = 0.248 +I0916 15:35:14.875610 140285881828544 spec.py:333] Evaluating on the training split. +I0916 15:35:17.074343 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 15:36:21.994919 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 15:36:24.178446 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 15:37:32.611151 140285881828544 spec.py:363] Evaluating on the test split. +I0916 15:37:34.789228 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 15:38:43.492561 140285881828544 submission_runner.py:516] Time since start: 2205.58s, Step: 8346, {'train/accuracy': 0.5981583541573957, 'train/loss': 2.107146575725419, 'train/bleu': 28.964552267912858, 'validation/accuracy': 0.6117965059329704, 'validation/loss': 2.0157274863299897, 'validation/bleu': 24.563140508826766, 'validation/num_examples': 3000, 'test/accuracy': 0.6157806054267619, 'test/loss': 1.9821425977572482, 'test/bleu': 23.708120918429476, 'test/num_examples': 3003, 'score': 1355.6129999160767, 'total_duration': 2205.5780696868896, 'accumulated_submission_time': 1355.6129999160767, 'accumulated_eval_time': 844.0106911659241, 'accumulated_logging_time': 0.26892995834350586} +I0916 15:38:43.517533 140257433470720 logging_writer.py:48] [8346] accumulated_eval_time=844.011, accumulated_logging_time=0.26893, accumulated_submission_time=1355.61, global_step=8346, preemption_count=0, score=1355.61, test/accuracy=0.615781, test/bleu=23.7081, test/loss=1.98214, test/num_examples=3003, total_duration=2205.58, train/accuracy=0.598158, train/bleu=28.9646, train/loss=2.10715, validation/accuracy=0.611797, validation/bleu=24.5631, validation/loss=2.01573, validation/num_examples=3000 +I0916 15:39:07.921570 140257425078016 logging_writer.py:48] [8500] global_step=8500, grad_norm=0.266254, loss=2.24006 +I0916 15:39:07.924482 140285881828544 submission.py:307] 8500) loss = 2.240, grad_norm = 0.266 +I0916 15:40:24.720854 140257433470720 logging_writer.py:48] [9000] global_step=9000, grad_norm=0.241385, loss=2.176 +I0916 15:40:24.723988 140285881828544 submission.py:307] 9000) loss = 2.176, grad_norm = 0.241 +I0916 15:41:41.648451 140257425078016 logging_writer.py:48] [9500] global_step=9500, grad_norm=0.238309, loss=2.18419 +I0916 15:41:41.651788 140285881828544 submission.py:307] 9500) loss = 2.184, grad_norm = 0.238 +I0916 15:42:58.577864 140257433470720 logging_writer.py:48] [10000] global_step=10000, grad_norm=0.225902, loss=2.14899 +I0916 15:42:58.580912 140285881828544 submission.py:307] 10000) loss = 2.149, grad_norm = 0.226 +I0916 15:44:15.510171 140257425078016 logging_writer.py:48] [10500] global_step=10500, grad_norm=0.217071, loss=2.14728 +I0916 15:44:15.513377 140285881828544 submission.py:307] 10500) loss = 2.147, grad_norm = 0.217 +I0916 15:45:32.484834 140257433470720 logging_writer.py:48] [11000] global_step=11000, grad_norm=0.222573, loss=2.1405 +I0916 15:45:32.487899 140285881828544 submission.py:307] 11000) loss = 2.141, grad_norm = 0.223 +I0916 15:46:49.459397 140257425078016 logging_writer.py:48] [11500] global_step=11500, grad_norm=0.202438, loss=2.09444 +I0916 15:46:49.462439 140285881828544 submission.py:307] 11500) loss = 2.094, grad_norm = 0.202 +I0916 15:48:06.449832 140257433470720 logging_writer.py:48] [12000] global_step=12000, grad_norm=0.208924, loss=2.15045 +I0916 15:48:06.452850 140285881828544 submission.py:307] 12000) loss = 2.150, grad_norm = 0.209 +I0916 15:49:23.395149 140257425078016 logging_writer.py:48] [12500] global_step=12500, grad_norm=0.207512, loss=2.10153 +I0916 15:49:23.398120 140285881828544 submission.py:307] 12500) loss = 2.102, grad_norm = 0.208 +I0916 15:49:28.127569 140285881828544 spec.py:333] Evaluating on the training split. +I0916 15:49:30.326099 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 15:50:42.482315 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 15:50:44.658194 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 15:51:46.447692 140285881828544 spec.py:363] Evaluating on the test split. +I0916 15:51:48.622420 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 15:52:46.246883 140285881828544 submission_runner.py:516] Time since start: 3048.33s, Step: 12528, {'train/accuracy': 0.6432607033202835, 'train/loss': 1.76966246447445, 'train/bleu': 31.310136574467155, 'validation/accuracy': 0.6332469529206085, 'validation/loss': 1.830990161312321, 'validation/bleu': 26.32728614427261, 'validation/num_examples': 3000, 'test/accuracy': 0.6398117483005055, 'test/loss': 1.7749698593922492, 'test/bleu': 25.34061019767668, 'test/num_examples': 3003, 'score': 1997.7971255779266, 'total_duration': 3048.332426071167, 'accumulated_submission_time': 1997.7971255779266, 'accumulated_eval_time': 1042.1300301551819, 'accumulated_logging_time': 0.3031799793243408} +I0916 15:52:46.270859 140257433470720 logging_writer.py:48] [12528] accumulated_eval_time=1042.13, accumulated_logging_time=0.30318, accumulated_submission_time=1997.8, global_step=12528, preemption_count=0, score=1997.8, test/accuracy=0.639812, test/bleu=25.3406, test/loss=1.77497, test/num_examples=3003, total_duration=3048.33, train/accuracy=0.643261, train/bleu=31.3101, train/loss=1.76966, validation/accuracy=0.633247, validation/bleu=26.3273, validation/loss=1.83099, validation/num_examples=3000 +I0916 15:53:59.555275 140257425078016 logging_writer.py:48] [13000] global_step=13000, grad_norm=0.197909, loss=2.05395 +I0916 15:53:59.558563 140285881828544 submission.py:307] 13000) loss = 2.054, grad_norm = 0.198 +I0916 15:55:16.517254 140257433470720 logging_writer.py:48] [13500] global_step=13500, grad_norm=0.194832, loss=2.11368 +I0916 15:55:16.520231 140285881828544 submission.py:307] 13500) loss = 2.114, grad_norm = 0.195 +I0916 15:56:33.497357 140257425078016 logging_writer.py:48] [14000] global_step=14000, grad_norm=0.202421, loss=2.06431 +I0916 15:56:33.500378 140285881828544 submission.py:307] 14000) loss = 2.064, grad_norm = 0.202 +I0916 15:57:50.497263 140257433470720 logging_writer.py:48] [14500] global_step=14500, grad_norm=0.186592, loss=1.96158 +I0916 15:57:50.500219 140285881828544 submission.py:307] 14500) loss = 1.962, grad_norm = 0.187 +I0916 15:59:07.548577 140257425078016 logging_writer.py:48] [15000] global_step=15000, grad_norm=0.197675, loss=1.97787 +I0916 15:59:07.551767 140285881828544 submission.py:307] 15000) loss = 1.978, grad_norm = 0.198 +I0916 16:00:24.543813 140257433470720 logging_writer.py:48] [15500] global_step=15500, grad_norm=0.181347, loss=2.00335 +I0916 16:00:24.546752 140285881828544 submission.py:307] 15500) loss = 2.003, grad_norm = 0.181 +I0916 16:01:41.557791 140257425078016 logging_writer.py:48] [16000] global_step=16000, grad_norm=0.170643, loss=1.92736 +I0916 16:01:41.561041 140285881828544 submission.py:307] 16000) loss = 1.927, grad_norm = 0.171 +I0916 16:02:58.579855 140257433470720 logging_writer.py:48] [16500] global_step=16500, grad_norm=0.173613, loss=2.06939 +I0916 16:02:58.582800 140285881828544 submission.py:307] 16500) loss = 2.069, grad_norm = 0.174 +I0916 16:03:30.879480 140285881828544 spec.py:333] Evaluating on the training split. +I0916 16:03:33.078232 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 16:04:49.545519 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 16:04:51.726370 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 16:05:49.670818 140285881828544 spec.py:363] Evaluating on the test split. +I0916 16:05:51.854543 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 16:06:45.322321 140285881828544 submission_runner.py:516] Time since start: 3887.41s, Step: 16707, {'train/accuracy': 0.6300413768017097, 'train/loss': 1.8440618890101397, 'train/bleu': 30.73685338788265, 'validation/accuracy': 0.6441953602559175, 'validation/loss': 1.7323150053936096, 'validation/bleu': 27.362542756722107, 'validation/num_examples': 3000, 'test/accuracy': 0.654546510952298, 'test/loss': 1.6592146737551565, 'test/bleu': 26.680943990594606, 'test/num_examples': 3003, 'score': 2639.9789066314697, 'total_duration': 3887.4078834056854, 'accumulated_submission_time': 2639.9789066314697, 'accumulated_eval_time': 1236.5729298591614, 'accumulated_logging_time': 0.336212158203125} +I0916 16:06:45.346365 140257425078016 logging_writer.py:48] [16707] accumulated_eval_time=1236.57, accumulated_logging_time=0.336212, accumulated_submission_time=2639.98, global_step=16707, preemption_count=0, score=2639.98, test/accuracy=0.654547, test/bleu=26.6809, test/loss=1.65921, test/num_examples=3003, total_duration=3887.41, train/accuracy=0.630041, train/bleu=30.7369, train/loss=1.84406, validation/accuracy=0.644195, validation/bleu=27.3625, validation/loss=1.73232, validation/num_examples=3000 +I0916 16:07:31.108142 140257433470720 logging_writer.py:48] [17000] global_step=17000, grad_norm=0.167424, loss=2.04553 +I0916 16:07:31.111153 140285881828544 submission.py:307] 17000) loss = 2.046, grad_norm = 0.167 +I0916 16:08:48.043945 140257425078016 logging_writer.py:48] [17500] global_step=17500, grad_norm=0.179276, loss=1.98212 +I0916 16:08:48.047014 140285881828544 submission.py:307] 17500) loss = 1.982, grad_norm = 0.179 +I0916 16:10:04.978161 140257433470720 logging_writer.py:48] [18000] global_step=18000, grad_norm=0.18127, loss=1.91903 +I0916 16:10:04.981169 140285881828544 submission.py:307] 18000) loss = 1.919, grad_norm = 0.181 +I0916 16:11:21.949144 140257425078016 logging_writer.py:48] [18500] global_step=18500, grad_norm=0.185859, loss=1.9351 +I0916 16:11:21.952387 140285881828544 submission.py:307] 18500) loss = 1.935, grad_norm = 0.186 +I0916 16:12:38.870186 140257433470720 logging_writer.py:48] [19000] global_step=19000, grad_norm=0.158704, loss=1.90068 +I0916 16:12:38.873118 140285881828544 submission.py:307] 19000) loss = 1.901, grad_norm = 0.159 +I0916 16:13:55.786872 140257425078016 logging_writer.py:48] [19500] global_step=19500, grad_norm=0.159276, loss=1.86463 +I0916 16:13:55.789824 140285881828544 submission.py:307] 19500) loss = 1.865, grad_norm = 0.159 +I0916 16:15:12.699586 140257433470720 logging_writer.py:48] [20000] global_step=20000, grad_norm=0.161442, loss=1.9624 +I0916 16:15:12.702693 140285881828544 submission.py:307] 20000) loss = 1.962, grad_norm = 0.161 +I0916 16:16:29.648225 140257425078016 logging_writer.py:48] [20500] global_step=20500, grad_norm=0.159109, loss=1.88719 +I0916 16:16:29.651480 140285881828544 submission.py:307] 20500) loss = 1.887, grad_norm = 0.159 +I0916 16:17:30.038083 140285881828544 spec.py:333] Evaluating on the training split. +I0916 16:17:32.237236 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 16:18:53.590419 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 16:18:55.770924 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 16:20:03.289394 140285881828544 spec.py:363] Evaluating on the test split. +I0916 16:20:05.467889 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 16:21:03.596072 140285881828544 submission_runner.py:516] Time since start: 4745.68s, Step: 20890, {'train/accuracy': 0.639524875193665, 'train/loss': 1.760459882653354, 'train/bleu': 30.873449442281895, 'validation/accuracy': 0.6528375345624977, 'validation/loss': 1.6656911414613582, 'validation/bleu': 27.87425041864447, 'validation/num_examples': 3000, 'test/accuracy': 0.6632967288362094, 'test/loss': 1.5971380948230782, 'test/bleu': 27.197294463741816, 'test/num_examples': 3003, 'score': 3282.2432339191437, 'total_duration': 4745.681620597839, 'accumulated_submission_time': 3282.2432339191437, 'accumulated_eval_time': 1450.1309490203857, 'accumulated_logging_time': 0.369384765625} +I0916 16:21:03.621059 140257433470720 logging_writer.py:48] [20890] accumulated_eval_time=1450.13, accumulated_logging_time=0.369385, accumulated_submission_time=3282.24, global_step=20890, preemption_count=0, score=3282.24, test/accuracy=0.663297, test/bleu=27.1973, test/loss=1.59714, test/num_examples=3003, total_duration=4745.68, train/accuracy=0.639525, train/bleu=30.8734, train/loss=1.76046, validation/accuracy=0.652838, validation/bleu=27.8743, validation/loss=1.66569, validation/num_examples=3000 +I0916 16:21:21.286135 140257425078016 logging_writer.py:48] [21000] global_step=21000, grad_norm=0.158461, loss=1.80713 +I0916 16:21:21.289017 140285881828544 submission.py:307] 21000) loss = 1.807, grad_norm = 0.158 +I0916 16:22:38.077474 140257433470720 logging_writer.py:48] [21500] global_step=21500, grad_norm=0.166544, loss=1.87943 +I0916 16:22:38.080660 140285881828544 submission.py:307] 21500) loss = 1.879, grad_norm = 0.167 +I0916 16:23:54.971178 140257425078016 logging_writer.py:48] [22000] global_step=22000, grad_norm=0.151493, loss=1.88497 +I0916 16:23:54.974178 140285881828544 submission.py:307] 22000) loss = 1.885, grad_norm = 0.151 +I0916 16:25:11.922331 140257433470720 logging_writer.py:48] [22500] global_step=22500, grad_norm=0.147421, loss=1.8658 +I0916 16:25:11.925363 140285881828544 submission.py:307] 22500) loss = 1.866, grad_norm = 0.147 +I0916 16:26:28.895446 140257425078016 logging_writer.py:48] [23000] global_step=23000, grad_norm=0.147151, loss=1.86054 +I0916 16:26:28.898533 140285881828544 submission.py:307] 23000) loss = 1.861, grad_norm = 0.147 +I0916 16:27:45.880277 140257433470720 logging_writer.py:48] [23500] global_step=23500, grad_norm=0.149202, loss=1.76767 +I0916 16:27:45.883459 140285881828544 submission.py:307] 23500) loss = 1.768, grad_norm = 0.149 +I0916 16:29:02.894600 140257425078016 logging_writer.py:48] [24000] global_step=24000, grad_norm=0.145288, loss=1.84421 +I0916 16:29:02.897676 140285881828544 submission.py:307] 24000) loss = 1.844, grad_norm = 0.145 +I0916 16:30:19.875708 140257433470720 logging_writer.py:48] [24500] global_step=24500, grad_norm=0.140208, loss=1.78609 +I0916 16:30:19.878704 140285881828544 submission.py:307] 24500) loss = 1.786, grad_norm = 0.140 +I0916 16:31:36.869009 140257425078016 logging_writer.py:48] [25000] global_step=25000, grad_norm=0.16235, loss=1.79337 +I0916 16:31:36.872034 140285881828544 submission.py:307] 25000) loss = 1.793, grad_norm = 0.162 +I0916 16:31:48.215327 140285881828544 spec.py:333] Evaluating on the training split. +I0916 16:31:50.415518 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 16:32:59.525696 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 16:33:01.705331 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 16:34:15.292640 140285881828544 spec.py:363] Evaluating on the test split. +I0916 16:34:17.474560 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 16:35:20.333686 140285881828544 submission_runner.py:516] Time since start: 5602.42s, Step: 25071, {'train/accuracy': 0.657928301371111, 'train/loss': 1.637032452853824, 'train/bleu': 32.65569117177226, 'validation/accuracy': 0.6585907180320145, 'validation/loss': 1.6224818198162454, 'validation/bleu': 28.51985047537219, 'validation/num_examples': 3000, 'test/accuracy': 0.6708151763407123, 'test/loss': 1.542599209807681, 'test/bleu': 27.78879918576069, 'test/num_examples': 3003, 'score': 3924.4118206501007, 'total_duration': 5602.4192237854, 'accumulated_submission_time': 3924.4118206501007, 'accumulated_eval_time': 1662.2493290901184, 'accumulated_logging_time': 0.40337610244750977} +I0916 16:35:20.359727 140257433470720 logging_writer.py:48] [25071] accumulated_eval_time=1662.25, accumulated_logging_time=0.403376, accumulated_submission_time=3924.41, global_step=25071, preemption_count=0, score=3924.41, test/accuracy=0.670815, test/bleu=27.7888, test/loss=1.5426, test/num_examples=3003, total_duration=5602.42, train/accuracy=0.657928, train/bleu=32.6557, train/loss=1.63703, validation/accuracy=0.658591, validation/bleu=28.5199, validation/loss=1.62248, validation/num_examples=3000 +I0916 16:36:26.966781 140257425078016 logging_writer.py:48] [25500] global_step=25500, grad_norm=0.145919, loss=1.80479 +I0916 16:36:26.969709 140285881828544 submission.py:307] 25500) loss = 1.805, grad_norm = 0.146 +I0916 16:37:43.826188 140257433470720 logging_writer.py:48] [26000] global_step=26000, grad_norm=0.15588, loss=1.80181 +I0916 16:37:43.829198 140285881828544 submission.py:307] 26000) loss = 1.802, grad_norm = 0.156 +I0916 16:39:00.840414 140257425078016 logging_writer.py:48] [26500] global_step=26500, grad_norm=0.154388, loss=1.89222 +I0916 16:39:00.843935 140285881828544 submission.py:307] 26500) loss = 1.892, grad_norm = 0.154 +I0916 16:40:17.856697 140257433470720 logging_writer.py:48] [27000] global_step=27000, grad_norm=0.183119, loss=1.84389 +I0916 16:40:17.859693 140285881828544 submission.py:307] 27000) loss = 1.844, grad_norm = 0.183 +I0916 16:41:34.844772 140257425078016 logging_writer.py:48] [27500] global_step=27500, grad_norm=0.1564, loss=1.77757 +I0916 16:41:34.847751 140285881828544 submission.py:307] 27500) loss = 1.778, grad_norm = 0.156 +I0916 16:42:51.821498 140257433470720 logging_writer.py:48] [28000] global_step=28000, grad_norm=0.1602, loss=1.81947 +I0916 16:42:51.824585 140285881828544 submission.py:307] 28000) loss = 1.819, grad_norm = 0.160 +I0916 16:44:08.806706 140257425078016 logging_writer.py:48] [28500] global_step=28500, grad_norm=0.152704, loss=1.83133 +I0916 16:44:08.809849 140285881828544 submission.py:307] 28500) loss = 1.831, grad_norm = 0.153 +I0916 16:45:25.792637 140257433470720 logging_writer.py:48] [29000] global_step=29000, grad_norm=0.199651, loss=1.751 +I0916 16:45:25.795693 140285881828544 submission.py:307] 29000) loss = 1.751, grad_norm = 0.200 +I0916 16:46:04.996421 140285881828544 spec.py:333] Evaluating on the training split. +I0916 16:46:07.195179 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 16:47:36.969020 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 16:47:39.150197 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 16:49:03.471415 140285881828544 spec.py:363] Evaluating on the test split. +I0916 16:49:05.651584 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 16:50:22.460956 140285881828544 submission_runner.py:516] Time since start: 6504.55s, Step: 29252, {'train/accuracy': 0.6512682311847234, 'train/loss': 1.6789634393180584, 'train/bleu': 31.91883672766166, 'validation/accuracy': 0.6644058970130563, 'validation/loss': 1.5817602618070452, 'validation/bleu': 28.512950924444468, 'validation/num_examples': 3000, 'test/accuracy': 0.6761141130672245, 'test/loss': 1.5004787998954157, 'test/bleu': 28.09399740478748, 'test/num_examples': 3003, 'score': 4566.620882987976, 'total_duration': 6504.546481370926, 'accumulated_submission_time': 4566.620882987976, 'accumulated_eval_time': 1919.713871717453, 'accumulated_logging_time': 0.4384465217590332} +I0916 16:50:22.487989 140257425078016 logging_writer.py:48] [29252] accumulated_eval_time=1919.71, accumulated_logging_time=0.438447, accumulated_submission_time=4566.62, global_step=29252, preemption_count=0, score=4566.62, test/accuracy=0.676114, test/bleu=28.094, test/loss=1.50048, test/num_examples=3003, total_duration=6504.55, train/accuracy=0.651268, train/bleu=31.9188, train/loss=1.67896, validation/accuracy=0.664406, validation/bleu=28.513, validation/loss=1.58176, validation/num_examples=3000 +I0916 16:51:01.340927 140257433470720 logging_writer.py:48] [29500] global_step=29500, grad_norm=0.171047, loss=1.77819 +I0916 16:51:01.343940 140285881828544 submission.py:307] 29500) loss = 1.778, grad_norm = 0.171 +I0916 16:52:18.201001 140257425078016 logging_writer.py:48] [30000] global_step=30000, grad_norm=0.169515, loss=1.82336 +I0916 16:52:18.204068 140285881828544 submission.py:307] 30000) loss = 1.823, grad_norm = 0.170 +I0916 16:53:35.157285 140257433470720 logging_writer.py:48] [30500] global_step=30500, grad_norm=0.179233, loss=1.72218 +I0916 16:53:35.160613 140285881828544 submission.py:307] 30500) loss = 1.722, grad_norm = 0.179 +I0916 16:54:52.087225 140257425078016 logging_writer.py:48] [31000] global_step=31000, grad_norm=0.156632, loss=1.74806 +I0916 16:54:52.090248 140285881828544 submission.py:307] 31000) loss = 1.748, grad_norm = 0.157 +I0916 16:56:09.069770 140257433470720 logging_writer.py:48] [31500] global_step=31500, grad_norm=0.155871, loss=1.75438 +I0916 16:56:09.072753 140285881828544 submission.py:307] 31500) loss = 1.754, grad_norm = 0.156 +I0916 16:57:26.052267 140257425078016 logging_writer.py:48] [32000] global_step=32000, grad_norm=0.152762, loss=1.79016 +I0916 16:57:26.055347 140285881828544 submission.py:307] 32000) loss = 1.790, grad_norm = 0.153 +I0916 16:58:43.079753 140257433470720 logging_writer.py:48] [32500] global_step=32500, grad_norm=0.158982, loss=1.67957 +I0916 16:58:43.083070 140285881828544 submission.py:307] 32500) loss = 1.680, grad_norm = 0.159 +I0916 17:00:00.070199 140257425078016 logging_writer.py:48] [33000] global_step=33000, grad_norm=0.182205, loss=1.76058 +I0916 17:00:00.073173 140285881828544 submission.py:307] 33000) loss = 1.761, grad_norm = 0.182 +I0916 17:01:07.119474 140285881828544 spec.py:333] Evaluating on the training split. +I0916 17:01:09.318595 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 17:02:34.713606 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 17:02:36.896054 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 17:03:45.108674 140285881828544 spec.py:363] Evaluating on the test split. +I0916 17:03:47.292812 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 17:04:47.630662 140285881828544 submission_runner.py:516] Time since start: 7369.72s, Step: 33433, {'train/accuracy': 0.655044324422871, 'train/loss': 1.643601800893325, 'train/bleu': 32.472740982188675, 'validation/accuracy': 0.668435605262179, 'validation/loss': 1.550556274100755, 'validation/bleu': 28.778562761376325, 'validation/num_examples': 3000, 'test/accuracy': 0.6795886351751786, 'test/loss': 1.4645453852187555, 'test/bleu': 28.33979120931825, 'test/num_examples': 3003, 'score': 5208.822647094727, 'total_duration': 7369.7161860466, 'accumulated_submission_time': 5208.822647094727, 'accumulated_eval_time': 2140.2251958847046, 'accumulated_logging_time': 0.4747128486633301} +I0916 17:04:47.657111 140257433470720 logging_writer.py:48] [33433] accumulated_eval_time=2140.23, accumulated_logging_time=0.474713, accumulated_submission_time=5208.82, global_step=33433, preemption_count=0, score=5208.82, test/accuracy=0.679589, test/bleu=28.3398, test/loss=1.46455, test/num_examples=3003, total_duration=7369.72, train/accuracy=0.655044, train/bleu=32.4727, train/loss=1.6436, validation/accuracy=0.668436, validation/bleu=28.7786, validation/loss=1.55056, validation/num_examples=3000 +I0916 17:04:58.720171 140257425078016 logging_writer.py:48] [33500] global_step=33500, grad_norm=0.164481, loss=1.73382 +I0916 17:04:58.723114 140285881828544 submission.py:307] 33500) loss = 1.734, grad_norm = 0.164 +I0916 17:06:15.519642 140257433470720 logging_writer.py:48] [34000] global_step=34000, grad_norm=0.168042, loss=1.76365 +I0916 17:06:15.522689 140285881828544 submission.py:307] 34000) loss = 1.764, grad_norm = 0.168 +I0916 17:07:32.408349 140257425078016 logging_writer.py:48] [34500] global_step=34500, grad_norm=0.170732, loss=1.70149 +I0916 17:07:32.411476 140285881828544 submission.py:307] 34500) loss = 1.701, grad_norm = 0.171 +I0916 17:08:49.387351 140257433470720 logging_writer.py:48] [35000] global_step=35000, grad_norm=0.175784, loss=1.80838 +I0916 17:08:49.390600 140285881828544 submission.py:307] 35000) loss = 1.808, grad_norm = 0.176 +I0916 17:10:06.351180 140257425078016 logging_writer.py:48] [35500] global_step=35500, grad_norm=0.211509, loss=1.76367 +I0916 17:10:06.354228 140285881828544 submission.py:307] 35500) loss = 1.764, grad_norm = 0.212 +I0916 17:11:23.357882 140257433470720 logging_writer.py:48] [36000] global_step=36000, grad_norm=0.185859, loss=1.73883 +I0916 17:11:23.360990 140285881828544 submission.py:307] 36000) loss = 1.739, grad_norm = 0.186 +I0916 17:12:40.335551 140257425078016 logging_writer.py:48] [36500] global_step=36500, grad_norm=0.177665, loss=1.70194 +I0916 17:12:40.338659 140285881828544 submission.py:307] 36500) loss = 1.702, grad_norm = 0.178 +I0916 17:13:57.320605 140257433470720 logging_writer.py:48] [37000] global_step=37000, grad_norm=0.185866, loss=1.7881 +I0916 17:13:57.323778 140285881828544 submission.py:307] 37000) loss = 1.788, grad_norm = 0.186 +I0916 17:15:14.304448 140257425078016 logging_writer.py:48] [37500] global_step=37500, grad_norm=0.221699, loss=1.64876 +I0916 17:15:14.307435 140285881828544 submission.py:307] 37500) loss = 1.649, grad_norm = 0.222 +I0916 17:15:32.257844 140285881828544 spec.py:333] Evaluating on the training split. +I0916 17:15:34.456472 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 17:17:02.348932 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 17:17:04.534895 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 17:18:20.090264 140285881828544 spec.py:363] Evaluating on the test split. +I0916 17:18:22.270682 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 17:19:25.018948 140285881828544 submission_runner.py:516] Time since start: 8247.10s, Step: 37614, {'train/accuracy': 0.667604986499684, 'train/loss': 1.553395903946688, 'train/bleu': 33.290601624517265, 'validation/accuracy': 0.6705186544494178, 'validation/loss': 1.5284733760275757, 'validation/bleu': 28.94812452196805, 'validation/num_examples': 3000, 'test/accuracy': 0.6837603857997792, 'test/loss': 1.4456154494218814, 'test/bleu': 28.645861782241727, 'test/num_examples': 3003, 'score': 5850.994255065918, 'total_duration': 8247.104498386383, 'accumulated_submission_time': 5850.994255065918, 'accumulated_eval_time': 2372.986330986023, 'accumulated_logging_time': 0.5110347270965576} +I0916 17:19:25.047133 140257433470720 logging_writer.py:48] [37614] accumulated_eval_time=2372.99, accumulated_logging_time=0.511035, accumulated_submission_time=5850.99, global_step=37614, preemption_count=0, score=5850.99, test/accuracy=0.68376, test/bleu=28.6459, test/loss=1.44562, test/num_examples=3003, total_duration=8247.1, train/accuracy=0.667605, train/bleu=33.2906, train/loss=1.5534, validation/accuracy=0.670519, validation/bleu=28.9481, validation/loss=1.52847, validation/num_examples=3000 +I0916 17:20:25.104924 140257425078016 logging_writer.py:48] [38000] global_step=38000, grad_norm=0.182951, loss=1.79669 +I0916 17:20:25.107993 140285881828544 submission.py:307] 38000) loss = 1.797, grad_norm = 0.183 +I0916 17:21:41.972355 140257433470720 logging_writer.py:48] [38500] global_step=38500, grad_norm=0.197964, loss=1.68052 +I0916 17:21:41.975494 140285881828544 submission.py:307] 38500) loss = 1.681, grad_norm = 0.198 +I0916 17:22:58.922477 140257425078016 logging_writer.py:48] [39000] global_step=39000, grad_norm=0.205386, loss=1.77527 +I0916 17:22:58.925525 140285881828544 submission.py:307] 39000) loss = 1.775, grad_norm = 0.205 +I0916 17:24:15.904311 140257433470720 logging_writer.py:48] [39500] global_step=39500, grad_norm=0.198075, loss=1.80423 +I0916 17:24:15.907338 140285881828544 submission.py:307] 39500) loss = 1.804, grad_norm = 0.198 +I0916 17:25:32.894344 140257425078016 logging_writer.py:48] [40000] global_step=40000, grad_norm=0.185946, loss=1.68279 +I0916 17:25:32.897422 140285881828544 submission.py:307] 40000) loss = 1.683, grad_norm = 0.186 +I0916 17:26:49.911252 140257433470720 logging_writer.py:48] [40500] global_step=40500, grad_norm=0.203276, loss=1.74892 +I0916 17:26:49.914323 140285881828544 submission.py:307] 40500) loss = 1.749, grad_norm = 0.203 +I0916 17:28:06.930353 140257425078016 logging_writer.py:48] [41000] global_step=41000, grad_norm=0.230443, loss=1.68731 +I0916 17:28:06.933367 140285881828544 submission.py:307] 41000) loss = 1.687, grad_norm = 0.230 +I0916 17:29:23.945346 140257433470720 logging_writer.py:48] [41500] global_step=41500, grad_norm=0.195247, loss=1.63842 +I0916 17:29:23.948514 140285881828544 submission.py:307] 41500) loss = 1.638, grad_norm = 0.195 +I0916 17:30:09.625832 140285881828544 spec.py:333] Evaluating on the training split. +I0916 17:30:11.825705 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 17:32:01.620700 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 17:32:03.803879 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 17:33:53.252792 140285881828544 spec.py:363] Evaluating on the test split. +I0916 17:33:55.428756 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 17:35:32.914186 140285881828544 submission_runner.py:516] Time since start: 9215.00s, Step: 41794, {'train/accuracy': 0.6589405104958028, 'train/loss': 1.6103582696447487, 'train/bleu': 32.836647836182514, 'validation/accuracy': 0.6729612776035016, 'validation/loss': 1.5113276455034657, 'validation/bleu': 29.203161052740487, 'validation/num_examples': 3000, 'test/accuracy': 0.6867236069955261, 'test/loss': 1.4221527112021382, 'test/bleu': 29.011979674353288, 'test/num_examples': 3003, 'score': 6493.1392595767975, 'total_duration': 9214.9997112751, 'accumulated_submission_time': 6493.1392595767975, 'accumulated_eval_time': 2696.274794101715, 'accumulated_logging_time': 0.5483815670013428} +I0916 17:35:32.941336 140257425078016 logging_writer.py:48] [41794] accumulated_eval_time=2696.27, accumulated_logging_time=0.548382, accumulated_submission_time=6493.14, global_step=41794, preemption_count=0, score=6493.14, test/accuracy=0.686724, test/bleu=29.012, test/loss=1.42215, test/num_examples=3003, total_duration=9215, train/accuracy=0.658941, train/bleu=32.8366, train/loss=1.61036, validation/accuracy=0.672961, validation/bleu=29.2032, validation/loss=1.51133, validation/num_examples=3000 +I0916 17:36:05.328236 140257433470720 logging_writer.py:48] [42000] global_step=42000, grad_norm=0.193369, loss=1.68292 +I0916 17:36:05.331116 140285881828544 submission.py:307] 42000) loss = 1.683, grad_norm = 0.193 +I0916 17:37:22.122891 140257425078016 logging_writer.py:48] [42500] global_step=42500, grad_norm=0.233017, loss=1.60454 +I0916 17:37:22.126064 140285881828544 submission.py:307] 42500) loss = 1.605, grad_norm = 0.233 +I0916 17:38:39.086462 140257433470720 logging_writer.py:48] [43000] global_step=43000, grad_norm=0.227334, loss=1.74204 +I0916 17:38:39.089576 140285881828544 submission.py:307] 43000) loss = 1.742, grad_norm = 0.227 +I0916 17:39:56.092999 140257425078016 logging_writer.py:48] [43500] global_step=43500, grad_norm=0.22107, loss=1.62782 +I0916 17:39:56.096067 140285881828544 submission.py:307] 43500) loss = 1.628, grad_norm = 0.221 +I0916 17:41:13.088212 140257433470720 logging_writer.py:48] [44000] global_step=44000, grad_norm=0.211479, loss=1.74618 +I0916 17:41:13.091249 140285881828544 submission.py:307] 44000) loss = 1.746, grad_norm = 0.211 +I0916 17:42:30.105058 140257425078016 logging_writer.py:48] [44500] global_step=44500, grad_norm=0.211234, loss=1.67156 +I0916 17:42:30.108454 140285881828544 submission.py:307] 44500) loss = 1.672, grad_norm = 0.211 +I0916 17:43:47.112506 140257433470720 logging_writer.py:48] [45000] global_step=45000, grad_norm=0.209398, loss=1.66689 +I0916 17:43:47.115508 140285881828544 submission.py:307] 45000) loss = 1.667, grad_norm = 0.209 +I0916 17:45:04.126247 140257425078016 logging_writer.py:48] [45500] global_step=45500, grad_norm=0.20611, loss=1.65367 +I0916 17:45:04.129323 140285881828544 submission.py:307] 45500) loss = 1.654, grad_norm = 0.206 +I0916 17:46:17.525257 140285881828544 spec.py:333] Evaluating on the training split. +I0916 17:46:19.725374 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 17:48:04.434311 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 17:48:06.610561 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 17:49:24.481559 140285881828544 spec.py:363] Evaluating on the test split. +I0916 17:49:26.665201 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 17:50:34.899209 140285881828544 submission_runner.py:516] Time since start: 10116.98s, Step: 45974, {'train/accuracy': 0.6654793370616996, 'train/loss': 1.5585175773938493, 'train/bleu': 33.60134517601314, 'validation/accuracy': 0.6750443267907403, 'validation/loss': 1.4944295095535083, 'validation/bleu': 29.118524233218636, 'validation/num_examples': 3000, 'test/accuracy': 0.6901400267270932, 'test/loss': 1.4036893193306605, 'test/bleu': 29.25787622025507, 'test/num_examples': 3003, 'score': 7135.297115802765, 'total_duration': 10116.984761714935, 'accumulated_submission_time': 7135.297115802765, 'accumulated_eval_time': 2953.6487753391266, 'accumulated_logging_time': 0.5850110054016113} +I0916 17:50:34.928397 140257433470720 logging_writer.py:48] [45974] accumulated_eval_time=2953.65, accumulated_logging_time=0.585011, accumulated_submission_time=7135.3, global_step=45974, preemption_count=0, score=7135.3, test/accuracy=0.69014, test/bleu=29.2579, test/loss=1.40369, test/num_examples=3003, total_duration=10117, train/accuracy=0.665479, train/bleu=33.6013, train/loss=1.55852, validation/accuracy=0.675044, validation/bleu=29.1185, validation/loss=1.49443, validation/num_examples=3000 +I0916 17:50:39.734752 140257425078016 logging_writer.py:48] [46000] global_step=46000, grad_norm=0.237552, loss=1.62132 +I0916 17:50:39.737631 140285881828544 submission.py:307] 46000) loss = 1.621, grad_norm = 0.238 +I0916 17:51:56.492418 140257433470720 logging_writer.py:48] [46500] global_step=46500, grad_norm=0.23422, loss=1.72388 +I0916 17:51:56.495526 140285881828544 submission.py:307] 46500) loss = 1.724, grad_norm = 0.234 +I0916 17:53:13.373526 140257425078016 logging_writer.py:48] [47000] global_step=47000, grad_norm=0.23288, loss=1.62952 +I0916 17:53:13.376481 140285881828544 submission.py:307] 47000) loss = 1.630, grad_norm = 0.233 +I0916 17:54:30.322854 140257433470720 logging_writer.py:48] [47500] global_step=47500, grad_norm=0.219313, loss=1.57785 +I0916 17:54:30.325957 140285881828544 submission.py:307] 47500) loss = 1.578, grad_norm = 0.219 +I0916 17:55:47.272826 140257425078016 logging_writer.py:48] [48000] global_step=48000, grad_norm=0.273525, loss=1.72016 +I0916 17:55:47.275895 140285881828544 submission.py:307] 48000) loss = 1.720, grad_norm = 0.274 +I0916 17:57:04.246102 140257433470720 logging_writer.py:48] [48500] global_step=48500, grad_norm=0.235591, loss=1.63369 +I0916 17:57:04.249413 140285881828544 submission.py:307] 48500) loss = 1.634, grad_norm = 0.236 +I0916 17:58:21.255122 140257425078016 logging_writer.py:48] [49000] global_step=49000, grad_norm=0.218981, loss=1.61595 +I0916 17:58:21.258098 140285881828544 submission.py:307] 49000) loss = 1.616, grad_norm = 0.219 +I0916 17:59:38.233075 140257433470720 logging_writer.py:48] [49500] global_step=49500, grad_norm=0.216294, loss=1.69046 +I0916 17:59:38.236059 140285881828544 submission.py:307] 49500) loss = 1.690, grad_norm = 0.216 +I0916 18:00:55.198521 140257425078016 logging_writer.py:48] [50000] global_step=50000, grad_norm=0.260881, loss=1.58047 +I0916 18:00:55.201548 140285881828544 submission.py:307] 50000) loss = 1.580, grad_norm = 0.261 +I0916 18:01:19.485392 140285881828544 spec.py:333] Evaluating on the training split. +I0916 18:01:21.682709 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 18:02:51.054464 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 18:02:53.240756 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 18:04:09.230707 140285881828544 spec.py:363] Evaluating on the test split. +I0916 18:04:11.416204 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 18:05:12.178677 140285881828544 submission_runner.py:516] Time since start: 10994.26s, Step: 50155, {'train/accuracy': 0.6773328180237118, 'train/loss': 1.4851885486171894, 'train/bleu': 34.272661187629076, 'validation/accuracy': 0.6784292817200035, 'validation/loss': 1.4790820363355692, 'validation/bleu': 29.64243472164039, 'validation/num_examples': 3000, 'test/accuracy': 0.6913834175817791, 'test/loss': 1.3904068437336587, 'test/bleu': 29.116156602215078, 'test/num_examples': 3003, 'score': 7777.399891376495, 'total_duration': 10994.264209508896, 'accumulated_submission_time': 7777.399891376495, 'accumulated_eval_time': 3186.342133998871, 'accumulated_logging_time': 0.6584129333496094} +I0916 18:05:12.206967 140257433470720 logging_writer.py:48] [50155] accumulated_eval_time=3186.34, accumulated_logging_time=0.658413, accumulated_submission_time=7777.4, global_step=50155, preemption_count=0, score=7777.4, test/accuracy=0.691383, test/bleu=29.1162, test/loss=1.39041, test/num_examples=3003, total_duration=10994.3, train/accuracy=0.677333, train/bleu=34.2727, train/loss=1.48519, validation/accuracy=0.678429, validation/bleu=29.6424, validation/loss=1.47908, validation/num_examples=3000 +I0916 18:06:05.959619 140257425078016 logging_writer.py:48] [50500] global_step=50500, grad_norm=0.23643, loss=1.61648 +I0916 18:06:05.962649 140285881828544 submission.py:307] 50500) loss = 1.616, grad_norm = 0.236 +I0916 18:07:22.868185 140257433470720 logging_writer.py:48] [51000] global_step=51000, grad_norm=0.250402, loss=1.62369 +I0916 18:07:22.871283 140285881828544 submission.py:307] 51000) loss = 1.624, grad_norm = 0.250 +I0916 18:08:39.880530 140257425078016 logging_writer.py:48] [51500] global_step=51500, grad_norm=0.26717, loss=1.69921 +I0916 18:08:39.883618 140285881828544 submission.py:307] 51500) loss = 1.699, grad_norm = 0.267 +I0916 18:09:56.889540 140257433470720 logging_writer.py:48] [52000] global_step=52000, grad_norm=0.253709, loss=1.6125 +I0916 18:09:56.892553 140285881828544 submission.py:307] 52000) loss = 1.613, grad_norm = 0.254 +I0916 18:11:13.945768 140257425078016 logging_writer.py:48] [52500] global_step=52500, grad_norm=0.245089, loss=1.63131 +I0916 18:11:13.949004 140285881828544 submission.py:307] 52500) loss = 1.631, grad_norm = 0.245 +I0916 18:12:31.022196 140257433470720 logging_writer.py:48] [53000] global_step=53000, grad_norm=0.246538, loss=1.67127 +I0916 18:12:31.025141 140285881828544 submission.py:307] 53000) loss = 1.671, grad_norm = 0.247 +I0916 18:13:48.063028 140257425078016 logging_writer.py:48] [53500] global_step=53500, grad_norm=0.278951, loss=1.61592 +I0916 18:13:48.066065 140285881828544 submission.py:307] 53500) loss = 1.616, grad_norm = 0.279 +I0916 18:15:05.102518 140257433470720 logging_writer.py:48] [54000] global_step=54000, grad_norm=0.256038, loss=1.68874 +I0916 18:15:05.105572 140285881828544 submission.py:307] 54000) loss = 1.689, grad_norm = 0.256 +I0916 18:15:56.828215 140285881828544 spec.py:333] Evaluating on the training split. +I0916 18:15:59.030461 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 18:17:42.528257 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 18:17:44.711733 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 18:19:15.985958 140285881828544 spec.py:363] Evaluating on the test split. +I0916 18:19:18.159173 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 18:20:54.728036 140285881828544 submission_runner.py:516] Time since start: 11936.81s, Step: 54333, {'train/accuracy': 0.6660078153398764, 'train/loss': 1.5606366860914214, 'train/bleu': 33.61155742324531, 'validation/accuracy': 0.6800411650196526, 'validation/loss': 1.4692231032473249, 'validation/bleu': 29.7080990770268, 'validation/num_examples': 3000, 'test/accuracy': 0.6941723316483644, 'test/loss': 1.3778477427226774, 'test/bleu': 29.43885734242232, 'test/num_examples': 3003, 'score': 8419.607409238815, 'total_duration': 11936.813574552536, 'accumulated_submission_time': 8419.607409238815, 'accumulated_eval_time': 3484.241970062256, 'accumulated_logging_time': 0.6958703994750977} +I0916 18:20:54.757263 140257425078016 logging_writer.py:48] [54333] accumulated_eval_time=3484.24, accumulated_logging_time=0.69587, accumulated_submission_time=8419.61, global_step=54333, preemption_count=0, score=8419.61, test/accuracy=0.694172, test/bleu=29.4389, test/loss=1.37785, test/num_examples=3003, total_duration=11936.8, train/accuracy=0.666008, train/bleu=33.6116, train/loss=1.56064, validation/accuracy=0.680041, validation/bleu=29.7081, validation/loss=1.46922, validation/num_examples=3000 +I0916 18:21:21.165461 140257433470720 logging_writer.py:48] [54500] global_step=54500, grad_norm=0.267034, loss=1.52187 +I0916 18:21:21.168460 140285881828544 submission.py:307] 54500) loss = 1.522, grad_norm = 0.267 +I0916 18:22:37.939845 140257425078016 logging_writer.py:48] [55000] global_step=55000, grad_norm=0.250728, loss=1.57143 +I0916 18:22:37.943121 140285881828544 submission.py:307] 55000) loss = 1.571, grad_norm = 0.251 +I0916 18:23:54.900928 140257433470720 logging_writer.py:48] [55500] global_step=55500, grad_norm=0.333491, loss=1.64198 +I0916 18:23:54.903915 140285881828544 submission.py:307] 55500) loss = 1.642, grad_norm = 0.333 +I0916 18:25:11.877679 140257425078016 logging_writer.py:48] [56000] global_step=56000, grad_norm=0.291852, loss=1.63392 +I0916 18:25:11.880715 140285881828544 submission.py:307] 56000) loss = 1.634, grad_norm = 0.292 +I0916 18:26:28.915520 140257433470720 logging_writer.py:48] [56500] global_step=56500, grad_norm=0.287258, loss=1.5444 +I0916 18:26:28.918580 140285881828544 submission.py:307] 56500) loss = 1.544, grad_norm = 0.287 +I0916 18:27:45.981784 140257425078016 logging_writer.py:48] [57000] global_step=57000, grad_norm=0.280786, loss=1.62567 +I0916 18:27:45.984905 140285881828544 submission.py:307] 57000) loss = 1.626, grad_norm = 0.281 +I0916 18:29:03.113988 140257433470720 logging_writer.py:48] [57500] global_step=57500, grad_norm=0.30155, loss=1.63515 +I0916 18:29:03.117059 140285881828544 submission.py:307] 57500) loss = 1.635, grad_norm = 0.302 +I0916 18:30:20.179116 140257425078016 logging_writer.py:48] [58000] global_step=58000, grad_norm=0.260529, loss=1.61048 +I0916 18:30:20.182294 140285881828544 submission.py:307] 58000) loss = 1.610, grad_norm = 0.261 +I0916 18:31:37.259756 140257433470720 logging_writer.py:48] [58500] global_step=58500, grad_norm=0.289347, loss=1.59111 +I0916 18:31:37.262818 140285881828544 submission.py:307] 58500) loss = 1.591, grad_norm = 0.289 +I0916 18:31:39.366719 140285881828544 spec.py:333] Evaluating on the training split. +I0916 18:31:41.568136 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 18:33:29.265999 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 18:33:31.445617 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 18:35:11.975429 140285881828544 spec.py:363] Evaluating on the test split. +I0916 18:35:14.159164 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 18:36:52.259708 140285881828544 submission_runner.py:516] Time since start: 12894.35s, Step: 58511, {'train/accuracy': 0.6746003470745751, 'train/loss': 1.4995030376208154, 'train/bleu': 34.61035100910041, 'validation/accuracy': 0.681020694101747, 'validation/loss': 1.4591866111393534, 'validation/bleu': 29.625713245246452, 'validation/num_examples': 3000, 'test/accuracy': 0.695671372959154, 'test/loss': 1.3631363118064028, 'test/bleu': 29.329694632284877, 'test/num_examples': 3003, 'score': 9061.798766613007, 'total_duration': 12894.345242500305, 'accumulated_submission_time': 9061.798766613007, 'accumulated_eval_time': 3797.135009288788, 'accumulated_logging_time': 0.7342274188995361} +I0916 18:36:52.286347 140257425078016 logging_writer.py:48] [58511] accumulated_eval_time=3797.14, accumulated_logging_time=0.734227, accumulated_submission_time=9061.8, global_step=58511, preemption_count=0, score=9061.8, test/accuracy=0.695671, test/bleu=29.3297, test/loss=1.36314, test/num_examples=3003, total_duration=12894.3, train/accuracy=0.6746, train/bleu=34.6104, train/loss=1.4995, validation/accuracy=0.681021, validation/bleu=29.6257, validation/loss=1.45919, validation/num_examples=3000 +I0916 18:38:08.254701 140257433470720 logging_writer.py:48] [59000] global_step=59000, grad_norm=0.288953, loss=1.65027 +I0916 18:38:08.257837 140285881828544 submission.py:307] 59000) loss = 1.650, grad_norm = 0.289 +I0916 18:39:25.277923 140257425078016 logging_writer.py:48] [59500] global_step=59500, grad_norm=0.335218, loss=1.63316 +I0916 18:39:25.280963 140285881828544 submission.py:307] 59500) loss = 1.633, grad_norm = 0.335 +I0916 18:40:42.337683 140257433470720 logging_writer.py:48] [60000] global_step=60000, grad_norm=0.326313, loss=1.61692 +I0916 18:40:42.340738 140285881828544 submission.py:307] 60000) loss = 1.617, grad_norm = 0.326 +I0916 18:41:59.388960 140257425078016 logging_writer.py:48] [60500] global_step=60500, grad_norm=0.345555, loss=1.64184 +I0916 18:41:59.392169 140285881828544 submission.py:307] 60500) loss = 1.642, grad_norm = 0.346 +I0916 18:43:16.434999 140257433470720 logging_writer.py:48] [61000] global_step=61000, grad_norm=0.30501, loss=1.6154 +I0916 18:43:16.437937 140285881828544 submission.py:307] 61000) loss = 1.615, grad_norm = 0.305 +I0916 18:44:33.529367 140257425078016 logging_writer.py:48] [61500] global_step=61500, grad_norm=0.301701, loss=1.59512 +I0916 18:44:33.532309 140285881828544 submission.py:307] 61500) loss = 1.595, grad_norm = 0.302 +I0916 18:45:50.589058 140257433470720 logging_writer.py:48] [62000] global_step=62000, grad_norm=0.315575, loss=1.72039 +I0916 18:45:50.592229 140285881828544 submission.py:307] 62000) loss = 1.720, grad_norm = 0.316 +I0916 18:47:07.677042 140257425078016 logging_writer.py:48] [62500] global_step=62500, grad_norm=0.307963, loss=1.66865 +I0916 18:47:07.680133 140285881828544 submission.py:307] 62500) loss = 1.669, grad_norm = 0.308 +I0916 18:47:36.899473 140285881828544 spec.py:333] Evaluating on the training split. +I0916 18:47:39.099351 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 18:49:21.810176 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 18:49:23.993594 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 18:50:21.774711 140285881828544 spec.py:363] Evaluating on the test split. +I0916 18:50:23.963656 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 18:51:20.642699 140285881828544 submission_runner.py:516] Time since start: 13762.73s, Step: 62687, {'train/accuracy': 0.6824528540400152, 'train/loss': 1.450200463702495, 'train/bleu': 34.866682799736104, 'validation/accuracy': 0.6835005145627456, 'validation/loss': 1.4477991980880585, 'validation/bleu': 29.770633461405943, 'validation/num_examples': 3000, 'test/accuracy': 0.6993666840973796, 'test/loss': 1.3493543373423973, 'test/bleu': 29.72264275714696, 'test/num_examples': 3003, 'score': 9703.985414505005, 'total_duration': 13762.728229761124, 'accumulated_submission_time': 9703.985414505005, 'accumulated_eval_time': 4020.878353357315, 'accumulated_logging_time': 0.7698471546173096} +I0916 18:51:20.670618 140257433470720 logging_writer.py:48] [62687] accumulated_eval_time=4020.88, accumulated_logging_time=0.769847, accumulated_submission_time=9703.99, global_step=62687, preemption_count=0, score=9703.99, test/accuracy=0.699367, test/bleu=29.7226, test/loss=1.34935, test/num_examples=3003, total_duration=13762.7, train/accuracy=0.682453, train/bleu=34.8667, train/loss=1.4502, validation/accuracy=0.683501, validation/bleu=29.7706, validation/loss=1.4478, validation/num_examples=3000 +I0916 18:52:09.500530 140257425078016 logging_writer.py:48] [63000] global_step=63000, grad_norm=0.306278, loss=1.60542 +I0916 18:52:09.503679 140285881828544 submission.py:307] 63000) loss = 1.605, grad_norm = 0.306 +I0916 18:53:26.356177 140257433470720 logging_writer.py:48] [63500] global_step=63500, grad_norm=0.363104, loss=1.57008 +I0916 18:53:26.359199 140285881828544 submission.py:307] 63500) loss = 1.570, grad_norm = 0.363 +I0916 18:54:43.376437 140257425078016 logging_writer.py:48] [64000] global_step=64000, grad_norm=0.305029, loss=1.60999 +I0916 18:54:43.379379 140285881828544 submission.py:307] 64000) loss = 1.610, grad_norm = 0.305 +I0916 18:56:00.426110 140257433470720 logging_writer.py:48] [64500] global_step=64500, grad_norm=0.331848, loss=1.58971 +I0916 18:56:00.429076 140285881828544 submission.py:307] 64500) loss = 1.590, grad_norm = 0.332 +I0916 18:57:17.507678 140257425078016 logging_writer.py:48] [65000] global_step=65000, grad_norm=0.318693, loss=1.56252 +I0916 18:57:17.510879 140285881828544 submission.py:307] 65000) loss = 1.563, grad_norm = 0.319 +I0916 18:58:34.624979 140257433470720 logging_writer.py:48] [65500] global_step=65500, grad_norm=0.329946, loss=1.54522 +I0916 18:58:34.628025 140285881828544 submission.py:307] 65500) loss = 1.545, grad_norm = 0.330 +I0916 18:59:51.683473 140257425078016 logging_writer.py:48] [66000] global_step=66000, grad_norm=0.348535, loss=1.53565 +I0916 18:59:51.686476 140285881828544 submission.py:307] 66000) loss = 1.536, grad_norm = 0.349 +I0916 19:01:08.736543 140257433470720 logging_writer.py:48] [66500] global_step=66500, grad_norm=0.296019, loss=1.54596 +I0916 19:01:08.739645 140285881828544 submission.py:307] 66500) loss = 1.546, grad_norm = 0.296 +I0916 19:02:05.235275 140285881828544 spec.py:333] Evaluating on the training split. +I0916 19:02:07.433443 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 19:03:51.318226 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 19:03:53.496409 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 19:05:31.907492 140285881828544 spec.py:363] Evaluating on the test split. +I0916 19:05:34.091751 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 19:07:09.939128 140285881828544 submission_runner.py:516] Time since start: 14712.02s, Step: 66864, {'train/accuracy': 0.6756834169702802, 'train/loss': 1.4924265777880892, 'train/bleu': 34.11392448626766, 'validation/accuracy': 0.6843684517240952, 'validation/loss': 1.4377314176203644, 'validation/bleu': 29.954212329587886, 'validation/num_examples': 3000, 'test/accuracy': 0.6993434431468247, 'test/loss': 1.3424210897100692, 'test/bleu': 29.931385894568795, 'test/num_examples': 3003, 'score': 10346.117608308792, 'total_duration': 14712.024676322937, 'accumulated_submission_time': 10346.117608308792, 'accumulated_eval_time': 4325.582231521606, 'accumulated_logging_time': 0.8068163394927979} +I0916 19:07:09.966010 140257425078016 logging_writer.py:48] [66864] accumulated_eval_time=4325.58, accumulated_logging_time=0.806816, accumulated_submission_time=10346.1, global_step=66864, preemption_count=0, score=10346.1, test/accuracy=0.699343, test/bleu=29.9314, test/loss=1.34242, test/num_examples=3003, total_duration=14712, train/accuracy=0.675683, train/bleu=34.1139, train/loss=1.49243, validation/accuracy=0.684368, validation/bleu=29.9542, validation/loss=1.43773, validation/num_examples=3000 +I0916 19:07:31.625333 140257433470720 logging_writer.py:48] [67000] global_step=67000, grad_norm=0.310824, loss=1.64183 +I0916 19:07:31.628431 140285881828544 submission.py:307] 67000) loss = 1.642, grad_norm = 0.311 +I0916 19:08:48.555227 140257425078016 logging_writer.py:48] [67500] global_step=67500, grad_norm=0.341689, loss=1.54091 +I0916 19:08:48.558224 140285881828544 submission.py:307] 67500) loss = 1.541, grad_norm = 0.342 +I0916 19:10:05.552218 140257433470720 logging_writer.py:48] [68000] global_step=68000, grad_norm=0.341566, loss=1.49386 +I0916 19:10:05.555269 140285881828544 submission.py:307] 68000) loss = 1.494, grad_norm = 0.342 +I0916 19:11:22.567767 140257425078016 logging_writer.py:48] [68500] global_step=68500, grad_norm=0.310312, loss=1.59262 +I0916 19:11:22.570756 140285881828544 submission.py:307] 68500) loss = 1.593, grad_norm = 0.310 +I0916 19:12:39.675934 140257433470720 logging_writer.py:48] [69000] global_step=69000, grad_norm=0.331445, loss=1.57958 +I0916 19:12:39.679041 140285881828544 submission.py:307] 69000) loss = 1.580, grad_norm = 0.331 +I0916 19:13:56.854110 140257425078016 logging_writer.py:48] [69500] global_step=69500, grad_norm=0.311593, loss=1.53827 +I0916 19:13:56.857128 140285881828544 submission.py:307] 69500) loss = 1.538, grad_norm = 0.312 +I0916 19:15:14.033824 140257433470720 logging_writer.py:48] [70000] global_step=70000, grad_norm=0.30089, loss=1.4875 +I0916 19:15:14.037042 140285881828544 submission.py:307] 70000) loss = 1.487, grad_norm = 0.301 +I0916 19:16:31.198827 140257425078016 logging_writer.py:48] [70500] global_step=70500, grad_norm=0.33163, loss=1.54092 +I0916 19:16:31.201775 140285881828544 submission.py:307] 70500) loss = 1.541, grad_norm = 0.332 +I0916 19:17:48.370620 140257433470720 logging_writer.py:48] [71000] global_step=71000, grad_norm=0.338869, loss=1.5587 +I0916 19:17:48.373800 140285881828544 submission.py:307] 71000) loss = 1.559, grad_norm = 0.339 +I0916 19:17:54.508966 140285881828544 spec.py:333] Evaluating on the training split. +I0916 19:17:56.710289 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 19:19:32.207762 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 19:19:34.391548 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 19:20:59.242050 140285881828544 spec.py:363] Evaluating on the test split. +I0916 19:21:01.424680 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 19:22:30.494041 140285881828544 submission_runner.py:516] Time since start: 15632.58s, Step: 71037, {'train/accuracy': 0.6834053912230068, 'train/loss': 1.4317170691641665, 'train/bleu': 35.21908852731881, 'validation/accuracy': 0.6867242811620439, 'validation/loss': 1.428437515498878, 'validation/bleu': 30.275626025858756, 'validation/num_examples': 3000, 'test/accuracy': 0.7004009063970716, 'test/loss': 1.3316930306199524, 'test/bleu': 29.98900185874826, 'test/num_examples': 3003, 'score': 10988.23679113388, 'total_duration': 15632.579575777054, 'accumulated_submission_time': 10988.23679113388, 'accumulated_eval_time': 4601.567341327667, 'accumulated_logging_time': 0.8427951335906982} +I0916 19:22:30.521618 140257425078016 logging_writer.py:48] [71037] accumulated_eval_time=4601.57, accumulated_logging_time=0.842795, accumulated_submission_time=10988.2, global_step=71037, preemption_count=0, score=10988.2, test/accuracy=0.700401, test/bleu=29.989, test/loss=1.33169, test/num_examples=3003, total_duration=15632.6, train/accuracy=0.683405, train/bleu=35.2191, train/loss=1.43172, validation/accuracy=0.686724, validation/bleu=30.2756, validation/loss=1.42844, validation/num_examples=3000 +I0916 19:23:42.453209 140257433470720 logging_writer.py:48] [71500] global_step=71500, grad_norm=0.333142, loss=1.47218 +I0916 19:23:42.456461 140285881828544 submission.py:307] 71500) loss = 1.472, grad_norm = 0.333 +I0916 19:24:59.298841 140257425078016 logging_writer.py:48] [72000] global_step=72000, grad_norm=0.316679, loss=1.53007 +I0916 19:24:59.302050 140285881828544 submission.py:307] 72000) loss = 1.530, grad_norm = 0.317 +I0916 19:26:16.252146 140257433470720 logging_writer.py:48] [72500] global_step=72500, grad_norm=0.328057, loss=1.59818 +I0916 19:26:16.255274 140285881828544 submission.py:307] 72500) loss = 1.598, grad_norm = 0.328 +I0916 19:27:33.229084 140257425078016 logging_writer.py:48] [73000] global_step=73000, grad_norm=0.332983, loss=1.59225 +I0916 19:27:33.232156 140285881828544 submission.py:307] 73000) loss = 1.592, grad_norm = 0.333 +I0916 19:28:50.215223 140257433470720 logging_writer.py:48] [73500] global_step=73500, grad_norm=0.336326, loss=1.53332 +I0916 19:28:50.218387 140285881828544 submission.py:307] 73500) loss = 1.533, grad_norm = 0.336 +I0916 19:30:07.231374 140257425078016 logging_writer.py:48] [74000] global_step=74000, grad_norm=0.310914, loss=1.556 +I0916 19:30:07.234524 140285881828544 submission.py:307] 74000) loss = 1.556, grad_norm = 0.311 +I0916 19:31:24.236019 140257433470720 logging_writer.py:48] [74500] global_step=74500, grad_norm=0.342582, loss=1.66568 +I0916 19:31:24.239009 140285881828544 submission.py:307] 74500) loss = 1.666, grad_norm = 0.343 +I0916 19:32:41.228347 140257425078016 logging_writer.py:48] [75000] global_step=75000, grad_norm=0.341484, loss=1.55394 +I0916 19:32:41.231541 140285881828544 submission.py:307] 75000) loss = 1.554, grad_norm = 0.341 +I0916 19:33:15.195314 140285881828544 spec.py:333] Evaluating on the training split. +I0916 19:33:17.394961 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 19:34:49.165324 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 19:34:51.346692 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 19:35:54.407502 140285881828544 spec.py:363] Evaluating on the test split. +I0916 19:35:56.581463 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 19:37:00.534754 140285881828544 submission_runner.py:516] Time since start: 16502.62s, Step: 75218, {'train/accuracy': 0.688193824532491, 'train/loss': 1.4156583858351073, 'train/bleu': 34.85227405520313, 'validation/accuracy': 0.6874062317888185, 'validation/loss': 1.4217455843696916, 'validation/bleu': 30.21420412657871, 'validation/num_examples': 3000, 'test/accuracy': 0.7021672186392424, 'test/loss': 1.321363063593051, 'test/bleu': 30.028664457576266, 'test/num_examples': 3003, 'score': 11630.491575241089, 'total_duration': 16502.620292186737, 'accumulated_submission_time': 11630.491575241089, 'accumulated_eval_time': 4826.906831741333, 'accumulated_logging_time': 0.8794615268707275} +I0916 19:37:00.562422 140257433470720 logging_writer.py:48] [75218] accumulated_eval_time=4826.91, accumulated_logging_time=0.879462, accumulated_submission_time=11630.5, global_step=75218, preemption_count=0, score=11630.5, test/accuracy=0.702167, test/bleu=30.0287, test/loss=1.32136, test/num_examples=3003, total_duration=16502.6, train/accuracy=0.688194, train/bleu=34.8523, train/loss=1.41566, validation/accuracy=0.687406, validation/bleu=30.2142, validation/loss=1.42175, validation/num_examples=3000 +I0916 19:37:44.562813 140257425078016 logging_writer.py:48] [75500] global_step=75500, grad_norm=0.37053, loss=1.51048 +I0916 19:37:44.565979 140285881828544 submission.py:307] 75500) loss = 1.510, grad_norm = 0.371 +I0916 19:39:01.386636 140257433470720 logging_writer.py:48] [76000] global_step=76000, grad_norm=0.352311, loss=1.55069 +I0916 19:39:01.389905 140285881828544 submission.py:307] 76000) loss = 1.551, grad_norm = 0.352 +I0916 19:40:18.233173 140257425078016 logging_writer.py:48] [76500] global_step=76500, grad_norm=0.35309, loss=1.52014 +I0916 19:40:18.236533 140285881828544 submission.py:307] 76500) loss = 1.520, grad_norm = 0.353 +I0916 19:41:35.171364 140257433470720 logging_writer.py:48] [77000] global_step=77000, grad_norm=0.335743, loss=1.53527 +I0916 19:41:35.174410 140285881828544 submission.py:307] 77000) loss = 1.535, grad_norm = 0.336 +I0916 19:42:52.105969 140257425078016 logging_writer.py:48] [77500] global_step=77500, grad_norm=0.32204, loss=1.51929 +I0916 19:42:52.109001 140285881828544 submission.py:307] 77500) loss = 1.519, grad_norm = 0.322 +I0916 19:44:09.080984 140257433470720 logging_writer.py:48] [78000] global_step=78000, grad_norm=0.377356, loss=1.57679 +I0916 19:44:09.084102 140285881828544 submission.py:307] 78000) loss = 1.577, grad_norm = 0.377 +I0916 19:45:26.073179 140257425078016 logging_writer.py:48] [78500] global_step=78500, grad_norm=0.318894, loss=1.54641 +I0916 19:45:26.076424 140285881828544 submission.py:307] 78500) loss = 1.546, grad_norm = 0.319 +I0916 19:46:43.022755 140257433470720 logging_writer.py:48] [79000] global_step=79000, grad_norm=0.370028, loss=1.52484 +I0916 19:46:43.025742 140285881828544 submission.py:307] 79000) loss = 1.525, grad_norm = 0.370 +I0916 19:47:45.128136 140285881828544 spec.py:333] Evaluating on the training split. +I0916 19:47:47.339159 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 19:49:07.680688 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 19:49:09.856330 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 19:50:13.379276 140285881828544 spec.py:363] Evaluating on the test split. +I0916 19:50:15.555173 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 19:51:14.425547 140285881828544 submission_runner.py:516] Time since start: 17356.51s, Step: 79401, {'train/accuracy': 0.686185196964878, 'train/loss': 1.428450612533509, 'train/bleu': 34.957344012463935, 'validation/accuracy': 0.6875674201187834, 'validation/loss': 1.4141449346567307, 'validation/bleu': 30.02909519582649, 'validation/num_examples': 3000, 'test/accuracy': 0.7039451513566904, 'test/loss': 1.3115709066875836, 'test/bleu': 30.37841595753352, 'test/num_examples': 3003, 'score': 12272.63627243042, 'total_duration': 17356.511085510254, 'accumulated_submission_time': 12272.63627243042, 'accumulated_eval_time': 5036.204295396805, 'accumulated_logging_time': 0.9161226749420166} +I0916 19:51:14.454559 140257425078016 logging_writer.py:48] [79401] accumulated_eval_time=5036.2, accumulated_logging_time=0.916123, accumulated_submission_time=12272.6, global_step=79401, preemption_count=0, score=12272.6, test/accuracy=0.703945, test/bleu=30.3784, test/loss=1.31157, test/num_examples=3003, total_duration=17356.5, train/accuracy=0.686185, train/bleu=34.9573, train/loss=1.42845, validation/accuracy=0.687567, validation/bleu=30.0291, validation/loss=1.41414, validation/num_examples=3000 +I0916 19:51:30.407799 140257433470720 logging_writer.py:48] [79500] global_step=79500, grad_norm=0.347379, loss=1.45176 +I0916 19:51:30.410718 140285881828544 submission.py:307] 79500) loss = 1.452, grad_norm = 0.347 +I0916 19:52:47.138300 140257425078016 logging_writer.py:48] [80000] global_step=80000, grad_norm=0.366936, loss=1.50642 +I0916 19:52:47.141499 140285881828544 submission.py:307] 80000) loss = 1.506, grad_norm = 0.367 +I0916 19:54:04.010361 140257433470720 logging_writer.py:48] [80500] global_step=80500, grad_norm=0.367297, loss=1.55177 +I0916 19:54:04.013376 140285881828544 submission.py:307] 80500) loss = 1.552, grad_norm = 0.367 +I0916 19:55:20.900025 140257425078016 logging_writer.py:48] [81000] global_step=81000, grad_norm=0.337896, loss=1.50171 +I0916 19:55:20.903068 140285881828544 submission.py:307] 81000) loss = 1.502, grad_norm = 0.338 +I0916 19:56:37.785109 140257433470720 logging_writer.py:48] [81500] global_step=81500, grad_norm=0.344954, loss=1.48245 +I0916 19:56:37.788239 140285881828544 submission.py:307] 81500) loss = 1.482, grad_norm = 0.345 +I0916 19:57:54.649922 140257425078016 logging_writer.py:48] [82000] global_step=82000, grad_norm=0.36911, loss=1.45204 +I0916 19:57:54.653038 140285881828544 submission.py:307] 82000) loss = 1.452, grad_norm = 0.369 +I0916 19:59:11.529965 140257433470720 logging_writer.py:48] [82500] global_step=82500, grad_norm=0.327696, loss=1.49672 +I0916 19:59:11.532998 140285881828544 submission.py:307] 82500) loss = 1.497, grad_norm = 0.328 +I0916 20:00:28.423372 140257425078016 logging_writer.py:48] [83000] global_step=83000, grad_norm=0.332648, loss=1.47984 +I0916 20:00:28.426357 140285881828544 submission.py:307] 83000) loss = 1.480, grad_norm = 0.333 +I0916 20:01:45.335014 140257433470720 logging_writer.py:48] [83500] global_step=83500, grad_norm=0.342401, loss=1.4669 +I0916 20:01:45.338026 140285881828544 submission.py:307] 83500) loss = 1.467, grad_norm = 0.342 +I0916 20:01:59.128685 140285881828544 spec.py:333] Evaluating on the training split. +I0916 20:02:01.332295 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 20:03:37.270674 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 20:03:39.446932 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 20:04:44.669851 140285881828544 spec.py:363] Evaluating on the test split. +I0916 20:04:46.846083 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 20:05:45.940089 140285881828544 submission_runner.py:516] Time since start: 18228.03s, Step: 83587, {'train/accuracy': 0.6951212514202426, 'train/loss': 1.3669021962115386, 'train/bleu': 35.89579936838176, 'validation/accuracy': 0.6879641913925432, 'validation/loss': 1.4048900703339078, 'validation/bleu': 30.144622406580115, 'validation/num_examples': 3000, 'test/accuracy': 0.7051885422113764, 'test/loss': 1.3016268665388415, 'test/bleu': 30.36694753249224, 'test/num_examples': 3003, 'score': 12914.87961435318, 'total_duration': 18228.025629758835, 'accumulated_submission_time': 12914.87961435318, 'accumulated_eval_time': 5263.015841007233, 'accumulated_logging_time': 0.9543108940124512} +I0916 20:05:45.968361 140257425078016 logging_writer.py:48] [83587] accumulated_eval_time=5263.02, accumulated_logging_time=0.954311, accumulated_submission_time=12914.9, global_step=83587, preemption_count=0, score=12914.9, test/accuracy=0.705189, test/bleu=30.3669, test/loss=1.30163, test/num_examples=3003, total_duration=18228, train/accuracy=0.695121, train/bleu=35.8958, train/loss=1.3669, validation/accuracy=0.687964, validation/bleu=30.1446, validation/loss=1.40489, validation/num_examples=3000 +I0916 20:06:50.029831 140257433470720 logging_writer.py:48] [84000] global_step=84000, grad_norm=0.349035, loss=1.49543 +I0916 20:06:50.032779 140285881828544 submission.py:307] 84000) loss = 1.495, grad_norm = 0.349 +I0916 20:08:06.803336 140257425078016 logging_writer.py:48] [84500] global_step=84500, grad_norm=0.353584, loss=1.50154 +I0916 20:08:06.806525 140285881828544 submission.py:307] 84500) loss = 1.502, grad_norm = 0.354 +I0916 20:09:23.655308 140257433470720 logging_writer.py:48] [85000] global_step=85000, grad_norm=0.33827, loss=1.50084 +I0916 20:09:23.658402 140285881828544 submission.py:307] 85000) loss = 1.501, grad_norm = 0.338 +I0916 20:10:40.546836 140257425078016 logging_writer.py:48] [85500] global_step=85500, grad_norm=0.359093, loss=1.47548 +I0916 20:10:40.549838 140285881828544 submission.py:307] 85500) loss = 1.475, grad_norm = 0.359 +I0916 20:11:57.435704 140257433470720 logging_writer.py:48] [86000] global_step=86000, grad_norm=0.324779, loss=1.47913 +I0916 20:11:57.438764 140285881828544 submission.py:307] 86000) loss = 1.479, grad_norm = 0.325 +I0916 20:13:14.327638 140257425078016 logging_writer.py:48] [86500] global_step=86500, grad_norm=0.305725, loss=1.37792 +I0916 20:13:14.330800 140285881828544 submission.py:307] 86500) loss = 1.378, grad_norm = 0.306 +I0916 20:14:31.212939 140257433470720 logging_writer.py:48] [87000] global_step=87000, grad_norm=0.409612, loss=1.38561 +I0916 20:14:31.215951 140285881828544 submission.py:307] 87000) loss = 1.386, grad_norm = 0.410 +I0916 20:15:48.111016 140257425078016 logging_writer.py:48] [87500] global_step=87500, grad_norm=0.333924, loss=1.44514 +I0916 20:15:48.114073 140285881828544 submission.py:307] 87500) loss = 1.445, grad_norm = 0.334 +I0916 20:16:30.547823 140285881828544 spec.py:333] Evaluating on the training split. +I0916 20:16:32.750233 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 20:18:07.101152 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 20:18:09.287199 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 20:19:09.745259 140285881828544 spec.py:363] Evaluating on the test split. +I0916 20:19:11.924973 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 20:20:13.034337 140285881828544 submission_runner.py:516] Time since start: 19095.12s, Step: 87773, {'train/accuracy': 0.6960266867731396, 'train/loss': 1.3705178130783466, 'train/bleu': 36.28646213688107, 'validation/accuracy': 0.6907291912065566, 'validation/loss': 1.402832594295173, 'validation/bleu': 30.39228775289873, 'validation/num_examples': 3000, 'test/accuracy': 0.7062692464121783, 'test/loss': 1.2978454912555923, 'test/bleu': 30.380732761160026, 'test/num_examples': 3003, 'score': 13557.02722811699, 'total_duration': 19095.119864702225, 'accumulated_submission_time': 13557.02722811699, 'accumulated_eval_time': 5485.502369880676, 'accumulated_logging_time': 0.991753101348877} +I0916 20:20:13.065466 140257433470720 logging_writer.py:48] [87773] accumulated_eval_time=5485.5, accumulated_logging_time=0.991753, accumulated_submission_time=13557, global_step=87773, preemption_count=0, score=13557, test/accuracy=0.706269, test/bleu=30.3807, test/loss=1.29785, test/num_examples=3003, total_duration=19095.1, train/accuracy=0.696027, train/bleu=36.2865, train/loss=1.37052, validation/accuracy=0.690729, validation/bleu=30.3923, validation/loss=1.40283, validation/num_examples=3000 +I0916 20:20:48.717047 140257425078016 logging_writer.py:48] [88000] global_step=88000, grad_norm=0.363432, loss=1.47694 +I0916 20:20:48.720035 140285881828544 submission.py:307] 88000) loss = 1.477, grad_norm = 0.363 +I0916 20:22:05.642760 140257433470720 logging_writer.py:48] [88500] global_step=88500, grad_norm=0.349777, loss=1.47718 +I0916 20:22:05.645826 140285881828544 submission.py:307] 88500) loss = 1.477, grad_norm = 0.350 +I0916 20:23:22.659821 140257425078016 logging_writer.py:48] [89000] global_step=89000, grad_norm=0.355879, loss=1.39859 +I0916 20:23:22.663018 140285881828544 submission.py:307] 89000) loss = 1.399, grad_norm = 0.356 +I0916 20:24:39.687176 140257433470720 logging_writer.py:48] [89500] global_step=89500, grad_norm=0.33646, loss=1.42695 +I0916 20:24:39.690233 140285881828544 submission.py:307] 89500) loss = 1.427, grad_norm = 0.336 +I0916 20:25:56.734830 140257425078016 logging_writer.py:48] [90000] global_step=90000, grad_norm=0.351224, loss=1.47634 +I0916 20:25:56.738030 140285881828544 submission.py:307] 90000) loss = 1.476, grad_norm = 0.351 +I0916 20:27:13.762117 140257433470720 logging_writer.py:48] [90500] global_step=90500, grad_norm=0.360872, loss=1.47643 +I0916 20:27:13.765156 140285881828544 submission.py:307] 90500) loss = 1.476, grad_norm = 0.361 +I0916 20:28:30.815384 140257425078016 logging_writer.py:48] [91000] global_step=91000, grad_norm=0.337169, loss=1.48209 +I0916 20:28:30.818655 140285881828544 submission.py:307] 91000) loss = 1.482, grad_norm = 0.337 +I0916 20:29:47.851061 140257433470720 logging_writer.py:48] [91500] global_step=91500, grad_norm=0.333389, loss=1.40945 +I0916 20:29:47.854120 140285881828544 submission.py:307] 91500) loss = 1.409, grad_norm = 0.333 +I0916 20:30:57.726455 140285881828544 spec.py:333] Evaluating on the training split. +I0916 20:30:59.921432 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 20:32:24.097441 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 20:32:26.283220 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 20:33:45.848613 140285881828544 spec.py:363] Evaluating on the test split. +I0916 20:33:48.032567 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 20:34:59.472364 140285881828544 submission_runner.py:516] Time since start: 19981.56s, Step: 91951, {'train/accuracy': 0.6927327011290452, 'train/loss': 1.3787650031743672, 'train/bleu': 35.77508179254608, 'validation/accuracy': 0.6903944154443218, 'validation/loss': 1.3989227117456695, 'validation/bleu': 30.246630583478655, 'validation/num_examples': 3000, 'test/accuracy': 0.7078728720004648, 'test/loss': 1.290620642321771, 'test/bleu': 30.569424982767647, 'test/num_examples': 3003, 'score': 14199.259328126907, 'total_duration': 19981.557898521423, 'accumulated_submission_time': 14199.259328126907, 'accumulated_eval_time': 5727.248344421387, 'accumulated_logging_time': 1.0326135158538818} +I0916 20:34:59.503151 140257425078016 logging_writer.py:48] [91951] accumulated_eval_time=5727.25, accumulated_logging_time=1.03261, accumulated_submission_time=14199.3, global_step=91951, preemption_count=0, score=14199.3, test/accuracy=0.707873, test/bleu=30.5694, test/loss=1.29062, test/num_examples=3003, total_duration=19981.6, train/accuracy=0.692733, train/bleu=35.7751, train/loss=1.37877, validation/accuracy=0.690394, validation/bleu=30.2466, validation/loss=1.39892, validation/num_examples=3000 +I0916 20:35:07.805746 140257433470720 logging_writer.py:48] [92000] global_step=92000, grad_norm=0.335915, loss=1.5433 +I0916 20:35:07.808617 140285881828544 submission.py:307] 92000) loss = 1.543, grad_norm = 0.336 +I0916 20:36:24.647563 140257425078016 logging_writer.py:48] [92500] global_step=92500, grad_norm=0.444347, loss=1.40757 +I0916 20:36:24.650646 140285881828544 submission.py:307] 92500) loss = 1.408, grad_norm = 0.444 +I0916 20:37:41.620407 140257433470720 logging_writer.py:48] [93000] global_step=93000, grad_norm=0.342994, loss=1.51056 +I0916 20:37:41.623556 140285881828544 submission.py:307] 93000) loss = 1.511, grad_norm = 0.343 +I0916 20:38:58.632689 140257425078016 logging_writer.py:48] [93500] global_step=93500, grad_norm=0.355798, loss=1.41584 +I0916 20:38:58.635716 140285881828544 submission.py:307] 93500) loss = 1.416, grad_norm = 0.356 +I0916 20:40:15.564224 140257433470720 logging_writer.py:48] [94000] global_step=94000, grad_norm=0.388112, loss=1.41961 +I0916 20:40:15.567203 140285881828544 submission.py:307] 94000) loss = 1.420, grad_norm = 0.388 +I0916 20:41:32.450892 140257425078016 logging_writer.py:48] [94500] global_step=94500, grad_norm=0.337661, loss=1.46746 +I0916 20:41:32.453858 140285881828544 submission.py:307] 94500) loss = 1.467, grad_norm = 0.338 +I0916 20:42:49.340750 140257433470720 logging_writer.py:48] [95000] global_step=95000, grad_norm=0.38235, loss=1.46602 +I0916 20:42:49.343895 140285881828544 submission.py:307] 95000) loss = 1.466, grad_norm = 0.382 +I0916 20:44:06.222031 140257425078016 logging_writer.py:48] [95500] global_step=95500, grad_norm=0.348948, loss=1.4407 +I0916 20:44:06.224967 140285881828544 submission.py:307] 95500) loss = 1.441, grad_norm = 0.349 +I0916 20:45:23.078994 140257433470720 logging_writer.py:48] [96000] global_step=96000, grad_norm=0.325074, loss=1.40009 +I0916 20:45:23.081982 140285881828544 submission.py:307] 96000) loss = 1.400, grad_norm = 0.325 +I0916 20:45:44.083726 140285881828544 spec.py:333] Evaluating on the training split. +I0916 20:45:46.282886 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 20:47:32.370184 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 20:47:34.550684 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 20:48:40.097981 140285881828544 spec.py:363] Evaluating on the test split. +I0916 20:48:42.277354 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 20:49:48.741173 140285881828544 submission_runner.py:516] Time since start: 20870.83s, Step: 96134, {'train/accuracy': 0.7004265985948693, 'train/loss': 1.3358186510400496, 'train/bleu': 36.00848370991645, 'validation/accuracy': 0.6915599310609912, 'validation/loss': 1.390955707306791, 'validation/bleu': 30.495445705351244, 'validation/num_examples': 3000, 'test/accuracy': 0.7087095462204405, 'test/loss': 1.2842161263145662, 'test/bleu': 30.535210420138235, 'test/num_examples': 3003, 'score': 14841.415359258652, 'total_duration': 20870.826719522476, 'accumulated_submission_time': 14841.415359258652, 'accumulated_eval_time': 5971.905826330185, 'accumulated_logging_time': 1.0726385116577148} +I0916 20:49:48.771581 140257425078016 logging_writer.py:48] [96134] accumulated_eval_time=5971.91, accumulated_logging_time=1.07264, accumulated_submission_time=14841.4, global_step=96134, preemption_count=0, score=14841.4, test/accuracy=0.70871, test/bleu=30.5352, test/loss=1.28422, test/num_examples=3003, total_duration=20870.8, train/accuracy=0.700427, train/bleu=36.0085, train/loss=1.33582, validation/accuracy=0.69156, validation/bleu=30.4954, validation/loss=1.39096, validation/num_examples=3000 +I0916 20:50:45.590987 140257433470720 logging_writer.py:48] [96500] global_step=96500, grad_norm=0.364058, loss=1.45794 +I0916 20:50:45.594288 140285881828544 submission.py:307] 96500) loss = 1.458, grad_norm = 0.364 +I0916 20:52:02.349940 140257425078016 logging_writer.py:48] [97000] global_step=97000, grad_norm=0.375773, loss=1.49947 +I0916 20:52:02.353099 140285881828544 submission.py:307] 97000) loss = 1.499, grad_norm = 0.376 +I0916 20:53:19.266798 140257433470720 logging_writer.py:48] [97500] global_step=97500, grad_norm=0.377015, loss=1.52844 +I0916 20:53:19.269719 140285881828544 submission.py:307] 97500) loss = 1.528, grad_norm = 0.377 +I0916 20:54:36.196938 140257425078016 logging_writer.py:48] [98000] global_step=98000, grad_norm=0.357726, loss=1.44223 +I0916 20:54:36.199984 140285881828544 submission.py:307] 98000) loss = 1.442, grad_norm = 0.358 +I0916 20:55:53.126348 140257433470720 logging_writer.py:48] [98500] global_step=98500, grad_norm=0.34772, loss=1.38909 +I0916 20:55:53.129437 140285881828544 submission.py:307] 98500) loss = 1.389, grad_norm = 0.348 +I0916 20:57:10.017778 140257425078016 logging_writer.py:48] [99000] global_step=99000, grad_norm=0.330447, loss=1.39345 +I0916 20:57:10.020875 140285881828544 submission.py:307] 99000) loss = 1.393, grad_norm = 0.330 +I0916 20:58:26.965931 140257433470720 logging_writer.py:48] [99500] global_step=99500, grad_norm=0.338683, loss=1.37643 +I0916 20:58:26.969054 140285881828544 submission.py:307] 99500) loss = 1.376, grad_norm = 0.339 +I0916 20:59:43.892493 140257425078016 logging_writer.py:48] [100000] global_step=100000, grad_norm=0.337371, loss=1.42999 +I0916 20:59:43.895584 140285881828544 submission.py:307] 100000) loss = 1.430, grad_norm = 0.337 +I0916 21:00:33.370411 140285881828544 spec.py:333] Evaluating on the training split. +I0916 21:00:35.571627 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 21:02:06.447508 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 21:02:08.627445 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 21:03:10.231450 140285881828544 spec.py:363] Evaluating on the test split. +I0916 21:03:12.418013 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 21:04:15.487135 140285881828544 submission_runner.py:516] Time since start: 21737.57s, Step: 100319, {'train/accuracy': 0.7012888725804972, 'train/loss': 1.3318683850678836, 'train/bleu': 36.78394909790258, 'validation/accuracy': 0.6915599310609912, 'validation/loss': 1.3895603239575456, 'validation/bleu': 30.66003052684969, 'validation/num_examples': 3000, 'test/accuracy': 0.7104526175120562, 'test/loss': 1.2800209531694846, 'test/bleu': 30.594078116323715, 'test/num_examples': 3003, 'score': 15483.591715574265, 'total_duration': 21737.572647094727, 'accumulated_submission_time': 15483.591715574265, 'accumulated_eval_time': 6194.022545337677, 'accumulated_logging_time': 1.1121065616607666} +I0916 21:04:15.517249 140257433470720 logging_writer.py:48] [100319] accumulated_eval_time=6194.02, accumulated_logging_time=1.11211, accumulated_submission_time=15483.6, global_step=100319, preemption_count=0, score=15483.6, test/accuracy=0.710453, test/bleu=30.5941, test/loss=1.28002, test/num_examples=3003, total_duration=21737.6, train/accuracy=0.701289, train/bleu=36.7839, train/loss=1.33187, validation/accuracy=0.69156, validation/bleu=30.66, validation/loss=1.38956, validation/num_examples=3000 +I0916 21:04:44.012015 140257425078016 logging_writer.py:48] [100500] global_step=100500, grad_norm=0.353307, loss=1.48939 +I0916 21:04:44.014918 140285881828544 submission.py:307] 100500) loss = 1.489, grad_norm = 0.353 +I0916 21:06:00.664542 140257433470720 logging_writer.py:48] [101000] global_step=101000, grad_norm=0.345977, loss=1.3875 +I0916 21:06:00.667539 140285881828544 submission.py:307] 101000) loss = 1.387, grad_norm = 0.346 +I0916 21:07:17.477413 140257425078016 logging_writer.py:48] [101500] global_step=101500, grad_norm=0.334682, loss=1.41652 +I0916 21:07:17.480371 140285881828544 submission.py:307] 101500) loss = 1.417, grad_norm = 0.335 +I0916 21:08:34.375512 140257433470720 logging_writer.py:48] [102000] global_step=102000, grad_norm=0.325127, loss=1.50947 +I0916 21:08:34.378757 140285881828544 submission.py:307] 102000) loss = 1.509, grad_norm = 0.325 +I0916 21:09:51.264786 140257425078016 logging_writer.py:48] [102500] global_step=102500, grad_norm=0.326146, loss=1.36531 +I0916 21:09:51.267799 140285881828544 submission.py:307] 102500) loss = 1.365, grad_norm = 0.326 +I0916 21:11:08.146167 140257433470720 logging_writer.py:48] [103000] global_step=103000, grad_norm=0.383153, loss=1.44073 +I0916 21:11:08.149352 140285881828544 submission.py:307] 103000) loss = 1.441, grad_norm = 0.383 +I0916 21:12:25.021943 140257425078016 logging_writer.py:48] [103500] global_step=103500, grad_norm=0.318099, loss=1.40587 +I0916 21:12:25.025007 140285881828544 submission.py:307] 103500) loss = 1.406, grad_norm = 0.318 +I0916 21:13:41.903607 140257433470720 logging_writer.py:48] [104000] global_step=104000, grad_norm=0.339557, loss=1.46587 +I0916 21:13:41.906773 140285881828544 submission.py:307] 104000) loss = 1.466, grad_norm = 0.340 +I0916 21:14:58.792186 140257425078016 logging_writer.py:48] [104500] global_step=104500, grad_norm=0.334603, loss=1.46793 +I0916 21:14:58.795214 140285881828544 submission.py:307] 104500) loss = 1.468, grad_norm = 0.335 +I0916 21:15:00.132989 140285881828544 spec.py:333] Evaluating on the training split. +I0916 21:15:02.335769 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 21:16:37.418754 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 21:16:39.605622 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 21:17:54.093231 140285881828544 spec.py:363] Evaluating on the test split. +I0916 21:17:56.276688 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 21:19:06.867180 140285881828544 submission_runner.py:516] Time since start: 22628.95s, Step: 104506, {'train/accuracy': 0.7011867710934027, 'train/loss': 1.3307043175781206, 'train/bleu': 35.85118838233687, 'validation/accuracy': 0.6919938996416659, 'validation/loss': 1.3865825051766252, 'validation/bleu': 30.470427481360787, 'validation/num_examples': 3000, 'test/accuracy': 0.7101969670559526, 'test/loss': 1.277310477311022, 'test/bleu': 30.73572516837829, 'test/num_examples': 3003, 'score': 16125.781217098236, 'total_duration': 22628.952724456787, 'accumulated_submission_time': 16125.781217098236, 'accumulated_eval_time': 6440.756788730621, 'accumulated_logging_time': 1.1513752937316895} +I0916 21:19:06.900145 140257433470720 logging_writer.py:48] [104506] accumulated_eval_time=6440.76, accumulated_logging_time=1.15138, accumulated_submission_time=16125.8, global_step=104506, preemption_count=0, score=16125.8, test/accuracy=0.710197, test/bleu=30.7357, test/loss=1.27731, test/num_examples=3003, total_duration=22629, train/accuracy=0.701187, train/bleu=35.8512, train/loss=1.3307, validation/accuracy=0.691994, validation/bleu=30.4704, validation/loss=1.38658, validation/num_examples=3000 +I0916 21:20:23.480329 140257425078016 logging_writer.py:48] [105000] global_step=105000, grad_norm=0.330535, loss=1.36625 +I0916 21:20:23.483513 140285881828544 submission.py:307] 105000) loss = 1.366, grad_norm = 0.331 +I0916 21:21:40.375437 140257433470720 logging_writer.py:48] [105500] global_step=105500, grad_norm=0.323275, loss=1.38532 +I0916 21:21:40.378649 140285881828544 submission.py:307] 105500) loss = 1.385, grad_norm = 0.323 +I0916 21:22:57.291497 140257425078016 logging_writer.py:48] [106000] global_step=106000, grad_norm=0.338747, loss=1.46751 +I0916 21:22:57.294545 140285881828544 submission.py:307] 106000) loss = 1.468, grad_norm = 0.339 +I0916 21:24:14.195342 140257433470720 logging_writer.py:48] [106500] global_step=106500, grad_norm=0.31748, loss=1.39374 +I0916 21:24:14.198407 140285881828544 submission.py:307] 106500) loss = 1.394, grad_norm = 0.317 +I0916 21:25:31.142900 140257425078016 logging_writer.py:48] [107000] global_step=107000, grad_norm=0.358426, loss=1.46419 +I0916 21:25:31.145890 140285881828544 submission.py:307] 107000) loss = 1.464, grad_norm = 0.358 +I0916 21:26:48.074969 140257433470720 logging_writer.py:48] [107500] global_step=107500, grad_norm=0.335765, loss=1.36895 +I0916 21:26:48.078063 140285881828544 submission.py:307] 107500) loss = 1.369, grad_norm = 0.336 +I0916 21:28:05.051023 140257425078016 logging_writer.py:48] [108000] global_step=108000, grad_norm=0.327309, loss=1.45801 +I0916 21:28:05.053969 140285881828544 submission.py:307] 108000) loss = 1.458, grad_norm = 0.327 +I0916 21:29:21.980131 140257433470720 logging_writer.py:48] [108500] global_step=108500, grad_norm=0.329663, loss=1.39635 +I0916 21:29:21.983236 140285881828544 submission.py:307] 108500) loss = 1.396, grad_norm = 0.330 +I0916 21:29:51.490653 140285881828544 spec.py:333] Evaluating on the training split. +I0916 21:29:53.689062 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 21:31:20.513900 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 21:31:22.696358 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 21:32:28.072367 140285881828544 spec.py:363] Evaluating on the test split. +I0916 21:32:30.255773 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 21:33:29.196050 140285881828544 submission_runner.py:516] Time since start: 23491.28s, Step: 108689, {'train/accuracy': 0.7037840941716911, 'train/loss': 1.316916271892047, 'train/bleu': 36.14055865113249, 'validation/accuracy': 0.6925146619384757, 'validation/loss': 1.3840584160456784, 'validation/bleu': 30.54382365888488, 'validation/num_examples': 3000, 'test/accuracy': 0.7104758584626111, 'test/loss': 1.2740839797222707, 'test/bleu': 30.683772847933387, 'test/num_examples': 3003, 'score': 16767.940527915955, 'total_duration': 23491.281604528427, 'accumulated_submission_time': 16767.940527915955, 'accumulated_eval_time': 6658.462338924408, 'accumulated_logging_time': 1.193418264389038} +I0916 21:33:29.226100 140257425078016 logging_writer.py:48] [108689] accumulated_eval_time=6658.46, accumulated_logging_time=1.19342, accumulated_submission_time=16767.9, global_step=108689, preemption_count=0, score=16767.9, test/accuracy=0.710476, test/bleu=30.6838, test/loss=1.27408, test/num_examples=3003, total_duration=23491.3, train/accuracy=0.703784, train/bleu=36.1406, train/loss=1.31692, validation/accuracy=0.692515, validation/bleu=30.5438, validation/loss=1.38406, validation/num_examples=3000 +I0916 21:34:17.669892 140257433470720 logging_writer.py:48] [109000] global_step=109000, grad_norm=0.359081, loss=1.41476 +I0916 21:34:17.672862 140285881828544 submission.py:307] 109000) loss = 1.415, grad_norm = 0.359 +I0916 21:35:34.441361 140257425078016 logging_writer.py:48] [109500] global_step=109500, grad_norm=0.322214, loss=1.36465 +I0916 21:35:34.444430 140285881828544 submission.py:307] 109500) loss = 1.365, grad_norm = 0.322 +I0916 21:36:51.306162 140257433470720 logging_writer.py:48] [110000] global_step=110000, grad_norm=0.340561, loss=1.45918 +I0916 21:36:51.309149 140285881828544 submission.py:307] 110000) loss = 1.459, grad_norm = 0.341 +I0916 21:38:08.263811 140257425078016 logging_writer.py:48] [110500] global_step=110500, grad_norm=0.337241, loss=1.43533 +I0916 21:38:08.266898 140285881828544 submission.py:307] 110500) loss = 1.435, grad_norm = 0.337 +I0916 21:39:25.207439 140257433470720 logging_writer.py:48] [111000] global_step=111000, grad_norm=0.33703, loss=1.45908 +I0916 21:39:25.210397 140285881828544 submission.py:307] 111000) loss = 1.459, grad_norm = 0.337 +I0916 21:40:42.099620 140257425078016 logging_writer.py:48] [111500] global_step=111500, grad_norm=0.3282, loss=1.47236 +I0916 21:40:42.102774 140285881828544 submission.py:307] 111500) loss = 1.472, grad_norm = 0.328 +I0916 21:41:59.025717 140257433470720 logging_writer.py:48] [112000] global_step=112000, grad_norm=0.359109, loss=1.37671 +I0916 21:41:59.028693 140285881828544 submission.py:307] 112000) loss = 1.377, grad_norm = 0.359 +I0916 21:43:15.970271 140257425078016 logging_writer.py:48] [112500] global_step=112500, grad_norm=0.343972, loss=1.42798 +I0916 21:43:15.973370 140285881828544 submission.py:307] 112500) loss = 1.428, grad_norm = 0.344 +I0916 21:44:13.902442 140285881828544 spec.py:333] Evaluating on the training split. +I0916 21:44:16.102435 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 21:45:58.061093 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 21:46:00.242652 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 21:47:06.534076 140285881828544 spec.py:363] Evaluating on the test split. +I0916 21:47:08.714119 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 21:48:09.584185 140285881828544 submission_runner.py:516] Time since start: 24371.67s, Step: 112874, {'train/accuracy': 0.7050293330295608, 'train/loss': 1.311290279518141, 'train/bleu': 36.558273890135176, 'validation/accuracy': 0.6926014556546106, 'validation/loss': 1.383067165937186, 'validation/bleu': 30.66006395991536, 'validation/num_examples': 3000, 'test/accuracy': 0.7111730869792574, 'test/loss': 1.272040954912556, 'test/bleu': 30.69741001883253, 'test/num_examples': 3003, 'score': 17410.203202724457, 'total_duration': 24371.669730186462, 'accumulated_submission_time': 17410.203202724457, 'accumulated_eval_time': 6894.1441440582275, 'accumulated_logging_time': 1.232969045639038} +I0916 21:48:09.613968 140257433470720 logging_writer.py:48] [112874] accumulated_eval_time=6894.14, accumulated_logging_time=1.23297, accumulated_submission_time=17410.2, global_step=112874, preemption_count=0, score=17410.2, test/accuracy=0.711173, test/bleu=30.6974, test/loss=1.27204, test/num_examples=3003, total_duration=24371.7, train/accuracy=0.705029, train/bleu=36.5583, train/loss=1.31129, validation/accuracy=0.692601, validation/bleu=30.6601, validation/loss=1.38307, validation/num_examples=3000 +I0916 21:48:29.713565 140257425078016 logging_writer.py:48] [113000] global_step=113000, grad_norm=0.337049, loss=1.40284 +I0916 21:48:29.716636 140285881828544 submission.py:307] 113000) loss = 1.403, grad_norm = 0.337 +I0916 21:49:46.376092 140257433470720 logging_writer.py:48] [113500] global_step=113500, grad_norm=0.334832, loss=1.35395 +I0916 21:49:46.379016 140285881828544 submission.py:307] 113500) loss = 1.354, grad_norm = 0.335 +I0916 21:51:03.219592 140257425078016 logging_writer.py:48] [114000] global_step=114000, grad_norm=0.347998, loss=1.45154 +I0916 21:51:03.222677 140285881828544 submission.py:307] 114000) loss = 1.452, grad_norm = 0.348 +I0916 21:52:20.115739 140257433470720 logging_writer.py:48] [114500] global_step=114500, grad_norm=0.334149, loss=1.43535 +I0916 21:52:20.118747 140285881828544 submission.py:307] 114500) loss = 1.435, grad_norm = 0.334 +I0916 21:53:36.997626 140257425078016 logging_writer.py:48] [115000] global_step=115000, grad_norm=0.337099, loss=1.41803 +I0916 21:53:37.000926 140285881828544 submission.py:307] 115000) loss = 1.418, grad_norm = 0.337 +I0916 21:54:53.870288 140257433470720 logging_writer.py:48] [115500] global_step=115500, grad_norm=0.348296, loss=1.34943 +I0916 21:54:53.873226 140285881828544 submission.py:307] 115500) loss = 1.349, grad_norm = 0.348 +I0916 21:56:10.758524 140257425078016 logging_writer.py:48] [116000] global_step=116000, grad_norm=0.338067, loss=1.35022 +I0916 21:56:10.761625 140285881828544 submission.py:307] 116000) loss = 1.350, grad_norm = 0.338 +I0916 21:57:27.657623 140257433470720 logging_writer.py:48] [116500] global_step=116500, grad_norm=0.336107, loss=1.37617 +I0916 21:57:27.660597 140285881828544 submission.py:307] 116500) loss = 1.376, grad_norm = 0.336 +I0916 21:58:44.575927 140257425078016 logging_writer.py:48] [117000] global_step=117000, grad_norm=0.345911, loss=1.38115 +I0916 21:58:44.579027 140285881828544 submission.py:307] 117000) loss = 1.381, grad_norm = 0.346 +I0916 21:58:54.218169 140285881828544 spec.py:333] Evaluating on the training split. +I0916 21:58:56.417867 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 22:00:32.528570 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 22:00:34.705785 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 22:01:41.587269 140285881828544 spec.py:363] Evaluating on the test split. +I0916 22:01:43.769618 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 22:02:45.970747 140285881828544 submission_runner.py:516] Time since start: 25248.06s, Step: 117060, {'train/accuracy': 0.7029898124061744, 'train/loss': 1.3130739653404078, 'train/bleu': 36.60939641849158, 'validation/accuracy': 0.6927006484730506, 'validation/loss': 1.3831094972473992, 'validation/bleu': 30.568080934996118, 'validation/num_examples': 3000, 'test/accuracy': 0.7111266050781477, 'test/loss': 1.272498420341642, 'test/bleu': 30.641968225785725, 'test/num_examples': 3003, 'score': 18052.374125003815, 'total_duration': 25248.056270599365, 'accumulated_submission_time': 18052.374125003815, 'accumulated_eval_time': 7125.89671421051, 'accumulated_logging_time': 1.2717673778533936} +I0916 22:02:46.002321 140257433470720 logging_writer.py:48] [117060] accumulated_eval_time=7125.9, accumulated_logging_time=1.27177, accumulated_submission_time=18052.4, global_step=117060, preemption_count=0, score=18052.4, test/accuracy=0.711127, test/bleu=30.642, test/loss=1.2725, test/num_examples=3003, total_duration=25248.1, train/accuracy=0.70299, train/bleu=36.6094, train/loss=1.31307, validation/accuracy=0.692701, validation/bleu=30.5681, validation/loss=1.38311, validation/num_examples=3000 +I0916 22:03:54.220843 140257425078016 logging_writer.py:48] [117500] global_step=117500, grad_norm=0.336188, loss=1.44833 +I0916 22:03:54.223913 140285881828544 submission.py:307] 117500) loss = 1.448, grad_norm = 0.336 +I0916 22:05:10.994363 140257433470720 logging_writer.py:48] [118000] global_step=118000, grad_norm=0.329952, loss=1.3407 +I0916 22:05:10.997397 140285881828544 submission.py:307] 118000) loss = 1.341, grad_norm = 0.330 +I0916 22:06:27.865595 140257425078016 logging_writer.py:48] [118500] global_step=118500, grad_norm=0.334645, loss=1.4023 +I0916 22:06:27.868541 140285881828544 submission.py:307] 118500) loss = 1.402, grad_norm = 0.335 +I0916 22:07:44.801110 140257433470720 logging_writer.py:48] [119000] global_step=119000, grad_norm=0.333125, loss=1.38155 +I0916 22:07:44.804306 140285881828544 submission.py:307] 119000) loss = 1.382, grad_norm = 0.333 +I0916 22:09:01.768193 140257425078016 logging_writer.py:48] [119500] global_step=119500, grad_norm=0.326852, loss=1.37375 +I0916 22:09:01.771268 140285881828544 submission.py:307] 119500) loss = 1.374, grad_norm = 0.327 +I0916 22:10:18.694165 140257433470720 logging_writer.py:48] [120000] global_step=120000, grad_norm=0.332532, loss=1.41581 +I0916 22:10:18.697166 140285881828544 submission.py:307] 120000) loss = 1.416, grad_norm = 0.333 +I0916 22:11:35.632192 140257425078016 logging_writer.py:48] [120500] global_step=120500, grad_norm=0.327216, loss=1.38321 +I0916 22:11:35.635229 140285881828544 submission.py:307] 120500) loss = 1.383, grad_norm = 0.327 +I0916 22:12:52.568256 140257433470720 logging_writer.py:48] [121000] global_step=121000, grad_norm=0.330397, loss=1.49795 +I0916 22:12:52.571439 140285881828544 submission.py:307] 121000) loss = 1.498, grad_norm = 0.330 +I0916 22:13:30.669562 140285881828544 spec.py:333] Evaluating on the training split. +I0916 22:13:32.869307 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 22:15:06.764026 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 22:15:08.952626 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 22:16:15.517132 140285881828544 spec.py:363] Evaluating on the test split. +I0916 22:16:17.700523 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 22:17:19.018634 140285881828544 submission_runner.py:516] Time since start: 26121.10s, Step: 121245, {'train/accuracy': 0.7058369118215073, 'train/loss': 1.2999912667848135, 'train/bleu': 36.67263580098895, 'validation/accuracy': 0.6928122403937955, 'validation/loss': 1.3830434332804304, 'validation/bleu': 30.556068910501633, 'validation/num_examples': 3000, 'test/accuracy': 0.7112311893556447, 'test/loss': 1.2723141268665388, 'test/bleu': 30.67450051412709, 'test/num_examples': 3003, 'score': 18694.610838890076, 'total_duration': 26121.104176282883, 'accumulated_submission_time': 18694.610838890076, 'accumulated_eval_time': 7354.245838403702, 'accumulated_logging_time': 1.312624216079712} +I0916 22:17:19.048977 140257425078016 logging_writer.py:48] [121245] accumulated_eval_time=7354.25, accumulated_logging_time=1.31262, accumulated_submission_time=18694.6, global_step=121245, preemption_count=0, score=18694.6, test/accuracy=0.711231, test/bleu=30.6745, test/loss=1.27231, test/num_examples=3003, total_duration=26121.1, train/accuracy=0.705837, train/bleu=36.6726, train/loss=1.29999, validation/accuracy=0.692812, validation/bleu=30.5561, validation/loss=1.38304, validation/num_examples=3000 +I0916 22:17:58.917278 140257433470720 logging_writer.py:48] [121500] global_step=121500, grad_norm=0.342836, loss=1.44095 +I0916 22:17:58.920432 140285881828544 submission.py:307] 121500) loss = 1.441, grad_norm = 0.343 +I0916 22:19:15.685834 140257425078016 logging_writer.py:48] [122000] global_step=122000, grad_norm=0.360044, loss=1.42174 +I0916 22:19:15.688963 140285881828544 submission.py:307] 122000) loss = 1.422, grad_norm = 0.360 +I0916 22:20:32.563089 140257433470720 logging_writer.py:48] [122500] global_step=122500, grad_norm=0.339157, loss=1.41113 +I0916 22:20:32.566210 140285881828544 submission.py:307] 122500) loss = 1.411, grad_norm = 0.339 +I0916 22:21:49.441819 140257425078016 logging_writer.py:48] [123000] global_step=123000, grad_norm=0.335838, loss=1.33121 +I0916 22:21:49.444853 140285881828544 submission.py:307] 123000) loss = 1.331, grad_norm = 0.336 +I0916 22:23:06.322486 140257433470720 logging_writer.py:48] [123500] global_step=123500, grad_norm=0.343205, loss=1.3751 +I0916 22:23:06.325764 140285881828544 submission.py:307] 123500) loss = 1.375, grad_norm = 0.343 +I0916 22:24:23.186950 140257425078016 logging_writer.py:48] [124000] global_step=124000, grad_norm=0.326112, loss=1.43972 +I0916 22:24:23.190056 140285881828544 submission.py:307] 124000) loss = 1.440, grad_norm = 0.326 +I0916 22:25:40.055709 140257433470720 logging_writer.py:48] [124500] global_step=124500, grad_norm=0.332107, loss=1.33837 +I0916 22:25:40.058842 140285881828544 submission.py:307] 124500) loss = 1.338, grad_norm = 0.332 +I0916 22:26:56.921519 140257425078016 logging_writer.py:48] [125000] global_step=125000, grad_norm=0.336576, loss=1.40196 +I0916 22:26:56.924515 140285881828544 submission.py:307] 125000) loss = 1.402, grad_norm = 0.337 +I0916 22:28:03.730282 140285881828544 spec.py:333] Evaluating on the training split. +I0916 22:28:05.930303 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 22:29:32.813892 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 22:29:35.000282 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 22:30:43.348188 140285881828544 spec.py:363] Evaluating on the test split. +I0916 22:30:45.529512 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 22:31:48.182377 140285881828544 submission_runner.py:516] Time since start: 26990.27s, Step: 125431, {'train/accuracy': 0.7039611557092633, 'train/loss': 1.313814891981494, 'train/bleu': 36.44912214972385, 'validation/accuracy': 0.6925766574500006, 'validation/loss': 1.382496613495183, 'validation/bleu': 30.57292623010075, 'validation/num_examples': 3000, 'test/accuracy': 0.7109406774737087, 'test/loss': 1.271411451978386, 'test/bleu': 30.791423307933886, 'test/num_examples': 3003, 'score': 19336.86342072487, 'total_duration': 26990.267922639847, 'accumulated_submission_time': 19336.86342072487, 'accumulated_eval_time': 7578.698003530502, 'accumulated_logging_time': 1.3521473407745361} +I0916 22:31:48.213901 140257433470720 logging_writer.py:48] [125431] accumulated_eval_time=7578.7, accumulated_logging_time=1.35215, accumulated_submission_time=19336.9, global_step=125431, preemption_count=0, score=19336.9, test/accuracy=0.710941, test/bleu=30.7914, test/loss=1.27141, test/num_examples=3003, total_duration=26990.3, train/accuracy=0.703961, train/bleu=36.4491, train/loss=1.31381, validation/accuracy=0.692577, validation/bleu=30.5729, validation/loss=1.3825, validation/num_examples=3000 +I0916 22:31:59.584164 140257425078016 logging_writer.py:48] [125500] global_step=125500, grad_norm=0.359102, loss=1.37414 +I0916 22:31:59.587052 140285881828544 submission.py:307] 125500) loss = 1.374, grad_norm = 0.359 +I0916 22:33:16.395350 140257433470720 logging_writer.py:48] [126000] global_step=126000, grad_norm=0.313277, loss=1.3458 +I0916 22:33:16.398481 140285881828544 submission.py:307] 126000) loss = 1.346, grad_norm = 0.313 +I0916 22:34:33.315870 140257425078016 logging_writer.py:48] [126500] global_step=126500, grad_norm=0.316124, loss=1.42347 +I0916 22:34:33.319113 140285881828544 submission.py:307] 126500) loss = 1.423, grad_norm = 0.316 +I0916 22:35:50.280129 140257433470720 logging_writer.py:48] [127000] global_step=127000, grad_norm=0.332204, loss=1.30975 +I0916 22:35:50.283142 140285881828544 submission.py:307] 127000) loss = 1.310, grad_norm = 0.332 +I0916 22:37:07.257882 140257425078016 logging_writer.py:48] [127500] global_step=127500, grad_norm=0.354182, loss=1.44107 +I0916 22:37:07.260880 140285881828544 submission.py:307] 127500) loss = 1.441, grad_norm = 0.354 +I0916 22:38:24.233580 140257433470720 logging_writer.py:48] [128000] global_step=128000, grad_norm=0.357379, loss=1.37685 +I0916 22:38:24.236493 140285881828544 submission.py:307] 128000) loss = 1.377, grad_norm = 0.357 +I0916 22:39:41.182356 140257425078016 logging_writer.py:48] [128500] global_step=128500, grad_norm=0.318741, loss=1.3781 +I0916 22:39:41.185472 140285881828544 submission.py:307] 128500) loss = 1.378, grad_norm = 0.319 +I0916 22:40:58.130121 140257433470720 logging_writer.py:48] [129000] global_step=129000, grad_norm=0.358778, loss=1.32647 +I0916 22:40:58.133275 140285881828544 submission.py:307] 129000) loss = 1.326, grad_norm = 0.359 +I0916 22:42:15.054738 140257425078016 logging_writer.py:48] [129500] global_step=129500, grad_norm=0.318183, loss=1.42899 +I0916 22:42:15.057730 140285881828544 submission.py:307] 129500) loss = 1.429, grad_norm = 0.318 +I0916 22:42:32.857627 140285881828544 spec.py:333] Evaluating on the training split. +I0916 22:42:35.053499 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 22:44:05.891636 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 22:44:08.071091 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 22:45:11.724766 140285881828544 spec.py:363] Evaluating on the test split. +I0916 22:45:13.909943 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 22:46:18.324594 140285881828544 submission_runner.py:516] Time since start: 27860.41s, Step: 129613, {'train/accuracy': 0.7036231554017125, 'train/loss': 1.3080286994443433, 'train/bleu': 36.59046463044769, 'validation/accuracy': 0.6921798861762408, 'validation/loss': 1.3833814056862284, 'validation/bleu': 30.45512799165426, 'validation/num_examples': 3000, 'test/accuracy': 0.7110917436523154, 'test/loss': 1.272092883911452, 'test/bleu': 30.75789742130872, 'test/num_examples': 3003, 'score': 19979.072958230972, 'total_duration': 27860.41013264656, 'accumulated_submission_time': 19979.072958230972, 'accumulated_eval_time': 7804.16498541832, 'accumulated_logging_time': 1.3926682472229004} +I0916 22:46:18.357221 140257433470720 logging_writer.py:48] [129613] accumulated_eval_time=7804.16, accumulated_logging_time=1.39267, accumulated_submission_time=19979.1, global_step=129613, preemption_count=0, score=19979.1, test/accuracy=0.711092, test/bleu=30.7579, test/loss=1.27209, test/num_examples=3003, total_duration=27860.4, train/accuracy=0.703623, train/bleu=36.5905, train/loss=1.30803, validation/accuracy=0.69218, validation/bleu=30.4551, validation/loss=1.38338, validation/num_examples=3000 +I0916 22:47:18.454174 140257425078016 logging_writer.py:48] [130000] global_step=130000, grad_norm=0.334388, loss=1.47901 +I0916 22:47:18.457238 140285881828544 submission.py:307] 130000) loss = 1.479, grad_norm = 0.334 +I0916 22:48:35.303615 140257433470720 logging_writer.py:48] [130500] global_step=130500, grad_norm=0.357075, loss=1.3796 +I0916 22:48:35.306537 140285881828544 submission.py:307] 130500) loss = 1.380, grad_norm = 0.357 +I0916 22:49:52.208661 140257425078016 logging_writer.py:48] [131000] global_step=131000, grad_norm=0.340468, loss=1.43419 +I0916 22:49:52.211891 140285881828544 submission.py:307] 131000) loss = 1.434, grad_norm = 0.340 +I0916 22:51:09.085734 140257433470720 logging_writer.py:48] [131500] global_step=131500, grad_norm=0.326297, loss=1.33728 +I0916 22:51:09.088844 140285881828544 submission.py:307] 131500) loss = 1.337, grad_norm = 0.326 +I0916 22:52:25.908737 140257425078016 logging_writer.py:48] [132000] global_step=132000, grad_norm=0.327592, loss=1.43214 +I0916 22:52:25.911828 140285881828544 submission.py:307] 132000) loss = 1.432, grad_norm = 0.328 +I0916 22:53:42.770487 140257433470720 logging_writer.py:48] [132500] global_step=132500, grad_norm=0.34307, loss=1.36622 +I0916 22:53:42.773393 140285881828544 submission.py:307] 132500) loss = 1.366, grad_norm = 0.343 +I0916 22:54:59.632166 140257425078016 logging_writer.py:48] [133000] global_step=133000, grad_norm=0.335175, loss=1.35799 +I0916 22:54:59.635439 140285881828544 submission.py:307] 133000) loss = 1.358, grad_norm = 0.335 +I0916 22:56:16.475786 140257433470720 logging_writer.py:48] [133500] global_step=133500, grad_norm=0.3419, loss=1.37612 +I0916 22:56:16.478781 140285881828544 submission.py:307] 133500) loss = 1.376, grad_norm = 0.342 +I0916 22:57:02.988751 140285881828544 spec.py:333] Evaluating on the training split. +I0916 22:57:05.185084 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 22:58:42.947572 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 22:58:45.128137 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 22:59:52.136878 140285881828544 spec.py:363] Evaluating on the test split. +I0916 22:59:54.322156 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 23:00:55.392876 140285881828544 submission_runner.py:516] Time since start: 28737.48s, Step: 133800, {'train/accuracy': 0.7033408232865306, 'train/loss': 1.3139929410993123, 'train/bleu': 36.58855409087423, 'validation/accuracy': 0.6931718143606402, 'validation/loss': 1.3845449839431625, 'validation/bleu': 30.54034889300614, 'validation/num_examples': 3000, 'test/accuracy': 0.7109174365231538, 'test/loss': 1.2735027743884726, 'test/bleu': 30.665177639086433, 'test/num_examples': 3003, 'score': 20621.282261371613, 'total_duration': 28737.478426218033, 'accumulated_submission_time': 20621.282261371613, 'accumulated_eval_time': 8036.569143533707, 'accumulated_logging_time': 1.4343180656433105} +I0916 23:00:55.425704 140257425078016 logging_writer.py:48] [133800] accumulated_eval_time=8036.57, accumulated_logging_time=1.43432, accumulated_submission_time=20621.3, global_step=133800, preemption_count=0, score=20621.3, test/accuracy=0.710917, test/bleu=30.6652, test/loss=1.2735, test/num_examples=3003, total_duration=28737.5, train/accuracy=0.703341, train/bleu=36.5886, train/loss=1.31399, validation/accuracy=0.693172, validation/bleu=30.5403, validation/loss=1.38454, validation/num_examples=3000 +I0916 23:01:26.833270 140257433470720 logging_writer.py:48] [134000] global_step=134000, grad_norm=0.356532, loss=1.3725 +I0916 23:01:26.836277 140285881828544 submission.py:307] 134000) loss = 1.372, grad_norm = 0.357 +I0916 23:02:43.571644 140257425078016 logging_writer.py:48] [134500] global_step=134500, grad_norm=0.333469, loss=1.35405 +I0916 23:02:43.574682 140285881828544 submission.py:307] 134500) loss = 1.354, grad_norm = 0.333 +I0916 23:04:00.377926 140257433470720 logging_writer.py:48] [135000] global_step=135000, grad_norm=0.343431, loss=1.40963 +I0916 23:04:00.381030 140285881828544 submission.py:307] 135000) loss = 1.410, grad_norm = 0.343 +I0916 23:05:17.208542 140257425078016 logging_writer.py:48] [135500] global_step=135500, grad_norm=0.319465, loss=1.36509 +I0916 23:05:17.211756 140285881828544 submission.py:307] 135500) loss = 1.365, grad_norm = 0.319 +I0916 23:06:34.040041 140257433470720 logging_writer.py:48] [136000] global_step=136000, grad_norm=0.367284, loss=1.43395 +I0916 23:06:34.043037 140285881828544 submission.py:307] 136000) loss = 1.434, grad_norm = 0.367 +I0916 23:07:50.878391 140257425078016 logging_writer.py:48] [136500] global_step=136500, grad_norm=0.335309, loss=1.42451 +I0916 23:07:50.881422 140285881828544 submission.py:307] 136500) loss = 1.425, grad_norm = 0.335 +I0916 23:09:07.732231 140257433470720 logging_writer.py:48] [137000] global_step=137000, grad_norm=0.326128, loss=1.42937 +I0916 23:09:07.735165 140285881828544 submission.py:307] 137000) loss = 1.429, grad_norm = 0.326 +I0916 23:10:24.538552 140257425078016 logging_writer.py:48] [137500] global_step=137500, grad_norm=0.328977, loss=1.42506 +I0916 23:10:24.541657 140285881828544 submission.py:307] 137500) loss = 1.425, grad_norm = 0.329 +I0916 23:11:40.041176 140285881828544 spec.py:333] Evaluating on the training split. +I0916 23:11:42.236152 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 23:13:16.888345 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 23:13:19.065046 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 23:14:23.893430 140285881828544 spec.py:363] Evaluating on the test split. +I0916 23:14:26.074296 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 23:15:19.216004 140285881828544 submission_runner.py:516] Time since start: 29601.30s, Step: 137989, {'train/accuracy': 0.7046032741999886, 'train/loss': 1.3126328895955734, 'train/bleu': 36.69733703822024, 'validation/accuracy': 0.6924526664269507, 'validation/loss': 1.3848173767219254, 'validation/bleu': 30.686698678834258, 'validation/num_examples': 3000, 'test/accuracy': 0.711266050781477, 'test/loss': 1.2750446662018478, 'test/bleu': 30.677366900510805, 'test/num_examples': 3003, 'score': 21263.457713603973, 'total_duration': 29601.30152130127, 'accumulated_submission_time': 21263.457713603973, 'accumulated_eval_time': 8255.743970632553, 'accumulated_logging_time': 1.4763462543487549} +I0916 23:15:19.248325 140257433470720 logging_writer.py:48] [137989] accumulated_eval_time=8255.74, accumulated_logging_time=1.47635, accumulated_submission_time=21263.5, global_step=137989, preemption_count=0, score=21263.5, test/accuracy=0.711266, test/bleu=30.6774, test/loss=1.27504, test/num_examples=3003, total_duration=29601.3, train/accuracy=0.704603, train/bleu=36.6973, train/loss=1.31263, validation/accuracy=0.692453, validation/bleu=30.6867, validation/loss=1.38482, validation/num_examples=3000 +I0916 23:15:21.713748 140257425078016 logging_writer.py:48] [138000] global_step=138000, grad_norm=0.338002, loss=1.32808 +I0916 23:15:21.716577 140285881828544 submission.py:307] 138000) loss = 1.328, grad_norm = 0.338 +I0916 23:16:38.275490 140257433470720 logging_writer.py:48] [138500] global_step=138500, grad_norm=0.32767, loss=1.4178 +I0916 23:16:38.278659 140285881828544 submission.py:307] 138500) loss = 1.418, grad_norm = 0.328 +I0916 23:17:55.046942 140257425078016 logging_writer.py:48] [139000] global_step=139000, grad_norm=0.335118, loss=1.46101 +I0916 23:17:55.049993 140285881828544 submission.py:307] 139000) loss = 1.461, grad_norm = 0.335 +I0916 23:19:11.849264 140257433470720 logging_writer.py:48] [139500] global_step=139500, grad_norm=0.357466, loss=1.3892 +I0916 23:19:11.852280 140285881828544 submission.py:307] 139500) loss = 1.389, grad_norm = 0.357 +I0916 23:20:28.660287 140257425078016 logging_writer.py:48] [140000] global_step=140000, grad_norm=0.356669, loss=1.31861 +I0916 23:20:28.663537 140285881828544 submission.py:307] 140000) loss = 1.319, grad_norm = 0.357 +I0916 23:21:45.492336 140257433470720 logging_writer.py:48] [140500] global_step=140500, grad_norm=0.336797, loss=1.40522 +I0916 23:21:45.495373 140285881828544 submission.py:307] 140500) loss = 1.405, grad_norm = 0.337 +I0916 23:23:02.321124 140257425078016 logging_writer.py:48] [141000] global_step=141000, grad_norm=0.338978, loss=1.39915 +I0916 23:23:02.324176 140285881828544 submission.py:307] 141000) loss = 1.399, grad_norm = 0.339 +I0916 23:24:19.160706 140257433470720 logging_writer.py:48] [141500] global_step=141500, grad_norm=0.34763, loss=1.49738 +I0916 23:24:19.163806 140285881828544 submission.py:307] 141500) loss = 1.497, grad_norm = 0.348 +I0916 23:25:35.988133 140257425078016 logging_writer.py:48] [142000] global_step=142000, grad_norm=0.395457, loss=1.46957 +I0916 23:25:35.991426 140285881828544 submission.py:307] 142000) loss = 1.470, grad_norm = 0.395 +I0916 23:26:03.897773 140285881828544 spec.py:333] Evaluating on the training split. +I0916 23:26:06.095865 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 23:27:35.118289 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 23:27:37.299337 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 23:28:47.541727 140285881828544 spec.py:363] Evaluating on the test split. +I0916 23:28:49.724864 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 23:29:56.949205 140285881828544 submission_runner.py:516] Time since start: 30479.03s, Step: 142179, {'train/accuracy': 0.7047295531748106, 'train/loss': 1.3150405869053976, 'train/bleu': 36.33011062295734, 'validation/accuracy': 0.6910639669687915, 'validation/loss': 1.3910401761912436, 'validation/bleu': 30.552253592969947, 'validation/num_examples': 3000, 'test/accuracy': 0.7107547498692697, 'test/loss': 1.2789763813839985, 'test/bleu': 30.876430129278646, 'test/num_examples': 3003, 'score': 21905.687613487244, 'total_duration': 30479.034685611725, 'accumulated_submission_time': 21905.687613487244, 'accumulated_eval_time': 8488.795366764069, 'accumulated_logging_time': 1.5180323123931885} +I0916 23:29:56.982016 140257433470720 logging_writer.py:48] [142179] accumulated_eval_time=8488.8, accumulated_logging_time=1.51803, accumulated_submission_time=21905.7, global_step=142179, preemption_count=0, score=21905.7, test/accuracy=0.710755, test/bleu=30.8764, test/loss=1.27898, test/num_examples=3003, total_duration=30479, train/accuracy=0.70473, train/bleu=36.3301, train/loss=1.31504, validation/accuracy=0.691064, validation/bleu=30.5523, validation/loss=1.39104, validation/num_examples=3000 +I0916 23:30:46.960156 140257425078016 logging_writer.py:48] [142500] global_step=142500, grad_norm=0.393652, loss=1.4677 +I0916 23:30:46.963197 140285881828544 submission.py:307] 142500) loss = 1.468, grad_norm = 0.394 +I0916 23:32:03.736770 140257433470720 logging_writer.py:48] [143000] global_step=143000, grad_norm=0.389493, loss=1.39296 +I0916 23:32:03.739845 140285881828544 submission.py:307] 143000) loss = 1.393, grad_norm = 0.389 +I0916 23:33:20.592827 140257425078016 logging_writer.py:48] [143500] global_step=143500, grad_norm=0.332702, loss=1.43622 +I0916 23:33:20.595875 140285881828544 submission.py:307] 143500) loss = 1.436, grad_norm = 0.333 +I0916 23:34:37.438547 140257433470720 logging_writer.py:48] [144000] global_step=144000, grad_norm=0.348983, loss=1.42637 +I0916 23:34:37.441587 140285881828544 submission.py:307] 144000) loss = 1.426, grad_norm = 0.349 +I0916 23:35:54.270415 140257425078016 logging_writer.py:48] [144500] global_step=144500, grad_norm=0.343364, loss=1.34759 +I0916 23:35:54.273552 140285881828544 submission.py:307] 144500) loss = 1.348, grad_norm = 0.343 +I0916 23:37:11.122873 140257433470720 logging_writer.py:48] [145000] global_step=145000, grad_norm=0.363445, loss=1.45079 +I0916 23:37:11.125953 140285881828544 submission.py:307] 145000) loss = 1.451, grad_norm = 0.363 +I0916 23:38:27.965337 140257425078016 logging_writer.py:48] [145500] global_step=145500, grad_norm=0.346569, loss=1.41205 +I0916 23:38:27.968405 140285881828544 submission.py:307] 145500) loss = 1.412, grad_norm = 0.347 +I0916 23:39:44.781889 140257433470720 logging_writer.py:48] [146000] global_step=146000, grad_norm=0.357945, loss=1.4652 +I0916 23:39:44.784850 140285881828544 submission.py:307] 146000) loss = 1.465, grad_norm = 0.358 +I0916 23:40:41.574042 140285881828544 spec.py:333] Evaluating on the training split. +I0916 23:40:43.770842 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 23:42:09.779214 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 23:42:11.964125 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 23:43:12.905765 140285881828544 spec.py:363] Evaluating on the test split. +I0916 23:43:15.086363 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 23:44:12.774718 140285881828544 submission_runner.py:516] Time since start: 31334.86s, Step: 146367, {'train/accuracy': 0.707080526550545, 'train/loss': 1.295063159760742, 'train/bleu': 36.38857651114236, 'validation/accuracy': 0.6911011642757064, 'validation/loss': 1.3908255167325885, 'validation/bleu': 30.476726665946735, 'validation/num_examples': 3000, 'test/accuracy': 0.7103945151356691, 'test/loss': 1.2809849986926964, 'test/bleu': 30.701723808290158, 'test/num_examples': 3003, 'score': 22547.857746839523, 'total_duration': 31334.860281705856, 'accumulated_submission_time': 22547.857746839523, 'accumulated_eval_time': 8699.996084451675, 'accumulated_logging_time': 1.5598969459533691} +I0916 23:44:12.808056 140257425078016 logging_writer.py:48] [146367] accumulated_eval_time=8700, accumulated_logging_time=1.5599, accumulated_submission_time=22547.9, global_step=146367, preemption_count=0, score=22547.9, test/accuracy=0.710395, test/bleu=30.7017, test/loss=1.28098, test/num_examples=3003, total_duration=31334.9, train/accuracy=0.707081, train/bleu=36.3886, train/loss=1.29506, validation/accuracy=0.691101, validation/bleu=30.4767, validation/loss=1.39083, validation/num_examples=3000 +I0916 23:44:33.955326 140257433470720 logging_writer.py:48] [146500] global_step=146500, grad_norm=0.371244, loss=1.3992 +I0916 23:44:33.958265 140285881828544 submission.py:307] 146500) loss = 1.399, grad_norm = 0.371 +I0916 23:45:50.555134 140257425078016 logging_writer.py:48] [147000] global_step=147000, grad_norm=0.362778, loss=1.49216 +I0916 23:45:50.558205 140285881828544 submission.py:307] 147000) loss = 1.492, grad_norm = 0.363 +I0916 23:47:07.310619 140257433470720 logging_writer.py:48] [147500] global_step=147500, grad_norm=0.373024, loss=1.33363 +I0916 23:47:07.313789 140285881828544 submission.py:307] 147500) loss = 1.334, grad_norm = 0.373 +I0916 23:48:24.156376 140257425078016 logging_writer.py:48] [148000] global_step=148000, grad_norm=0.360168, loss=1.43159 +I0916 23:48:24.159463 140285881828544 submission.py:307] 148000) loss = 1.432, grad_norm = 0.360 +I0916 23:49:40.974720 140257433470720 logging_writer.py:48] [148500] global_step=148500, grad_norm=0.362153, loss=1.37606 +I0916 23:49:40.977723 140285881828544 submission.py:307] 148500) loss = 1.376, grad_norm = 0.362 +I0916 23:50:57.811184 140257425078016 logging_writer.py:48] [149000] global_step=149000, grad_norm=0.333766, loss=1.43719 +I0916 23:50:57.814188 140285881828544 submission.py:307] 149000) loss = 1.437, grad_norm = 0.334 +I0916 23:52:14.648826 140257433470720 logging_writer.py:48] [149500] global_step=149500, grad_norm=0.341302, loss=1.43051 +I0916 23:52:14.651874 140285881828544 submission.py:307] 149500) loss = 1.431, grad_norm = 0.341 +I0916 23:53:31.462344 140257425078016 logging_writer.py:48] [150000] global_step=150000, grad_norm=0.345975, loss=1.43938 +I0916 23:53:31.465470 140285881828544 submission.py:307] 150000) loss = 1.439, grad_norm = 0.346 +I0916 23:54:48.289869 140257433470720 logging_writer.py:48] [150500] global_step=150500, grad_norm=0.355785, loss=1.4669 +I0916 23:54:48.292937 140285881828544 submission.py:307] 150500) loss = 1.467, grad_norm = 0.356 +I0916 23:54:57.463440 140285881828544 spec.py:333] Evaluating on the training split. +I0916 23:54:59.662732 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 23:56:28.579743 140285881828544 spec.py:346] Evaluating on the validation split. +I0916 23:56:30.760530 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 23:57:31.982276 140285881828544 spec.py:363] Evaluating on the test split. +I0916 23:57:34.165770 140285881828544 workload.py:152] Translating evaluation dataset. +I0916 23:58:27.417197 140285881828544 submission_runner.py:516] Time since start: 32189.50s, Step: 150557, {'train/accuracy': 0.7118764139948354, 'train/loss': 1.2636284464338765, 'train/bleu': 36.85572995214668, 'validation/accuracy': 0.6913739445264163, 'validation/loss': 1.3958163491463218, 'validation/bleu': 30.45687353008207, 'validation/num_examples': 3000, 'test/accuracy': 0.7088606123990471, 'test/loss': 1.289014202399628, 'test/bleu': 30.537840963292496, 'test/num_examples': 3003, 'score': 23190.091015338898, 'total_duration': 32189.50274205208, 'accumulated_submission_time': 23190.091015338898, 'accumulated_eval_time': 8909.949897766113, 'accumulated_logging_time': 1.6021971702575684} +I0916 23:58:27.449890 140257425078016 logging_writer.py:48] [150557] accumulated_eval_time=8909.95, accumulated_logging_time=1.6022, accumulated_submission_time=23190.1, global_step=150557, preemption_count=0, score=23190.1, test/accuracy=0.708861, test/bleu=30.5378, test/loss=1.28901, test/num_examples=3003, total_duration=32189.5, train/accuracy=0.711876, train/bleu=36.8557, train/loss=1.26363, validation/accuracy=0.691374, validation/bleu=30.4569, validation/loss=1.39582, validation/num_examples=3000 +I0916 23:59:36.075833 140257433470720 logging_writer.py:48] [151000] global_step=151000, grad_norm=0.336074, loss=1.47961 +I0916 23:59:36.078876 140285881828544 submission.py:307] 151000) loss = 1.480, grad_norm = 0.336 +I0917 00:00:52.825207 140257425078016 logging_writer.py:48] [151500] global_step=151500, grad_norm=0.36435, loss=1.51932 +I0917 00:00:52.828202 140285881828544 submission.py:307] 151500) loss = 1.519, grad_norm = 0.364 +I0917 00:02:09.632259 140257433470720 logging_writer.py:48] [152000] global_step=152000, grad_norm=0.349642, loss=1.45688 +I0917 00:02:09.635239 140285881828544 submission.py:307] 152000) loss = 1.457, grad_norm = 0.350 +I0917 00:03:26.478581 140257425078016 logging_writer.py:48] [152500] global_step=152500, grad_norm=0.367414, loss=1.44183 +I0917 00:03:26.481750 140285881828544 submission.py:307] 152500) loss = 1.442, grad_norm = 0.367 +I0917 00:04:43.341197 140257433470720 logging_writer.py:48] [153000] global_step=153000, grad_norm=0.344055, loss=1.41016 +I0917 00:04:43.344130 140285881828544 submission.py:307] 153000) loss = 1.410, grad_norm = 0.344 +I0917 00:06:00.163351 140257425078016 logging_writer.py:48] [153500] global_step=153500, grad_norm=0.350645, loss=1.41471 +I0917 00:06:00.166305 140285881828544 submission.py:307] 153500) loss = 1.415, grad_norm = 0.351 +I0917 00:07:16.993511 140257433470720 logging_writer.py:48] [154000] global_step=154000, grad_norm=0.352982, loss=1.43793 +I0917 00:07:16.996573 140285881828544 submission.py:307] 154000) loss = 1.438, grad_norm = 0.353 +I0917 00:08:33.895292 140257425078016 logging_writer.py:48] [154500] global_step=154500, grad_norm=0.359655, loss=1.4882 +I0917 00:08:33.898574 140285881828544 submission.py:307] 154500) loss = 1.488, grad_norm = 0.360 +I0917 00:09:12.114950 140285881828544 spec.py:333] Evaluating on the training split. +I0917 00:09:14.313537 140285881828544 workload.py:152] Translating evaluation dataset. +I0917 00:10:39.624841 140285881828544 spec.py:346] Evaluating on the validation split. +I0917 00:10:41.803610 140285881828544 workload.py:152] Translating evaluation dataset. +I0917 00:11:49.176789 140285881828544 spec.py:363] Evaluating on the test split. +I0917 00:11:51.357950 140285881828544 workload.py:152] Translating evaluation dataset. +I0917 00:12:52.029750 140285881828544 submission_runner.py:516] Time since start: 33054.12s, Step: 154746, {'train/accuracy': 0.7008275420629115, 'train/loss': 1.330113397694541, 'train/bleu': 35.98773863758199, 'validation/accuracy': 0.6904688100581517, 'validation/loss': 1.3992569062999838, 'validation/bleu': 30.408232851549812, 'validation/num_examples': 3000, 'test/accuracy': 0.7078844924757423, 'test/loss': 1.291251053105572, 'test/bleu': 30.342683619124177, 'test/num_examples': 3003, 'score': 23832.328326940536, 'total_duration': 33054.1152844429, 'accumulated_submission_time': 23832.328326940536, 'accumulated_eval_time': 9129.864770650864, 'accumulated_logging_time': 1.6439316272735596} +I0917 00:12:52.062655 140257433470720 logging_writer.py:48] [154746] accumulated_eval_time=9129.86, accumulated_logging_time=1.64393, accumulated_submission_time=23832.3, global_step=154746, preemption_count=0, score=23832.3, test/accuracy=0.707884, test/bleu=30.3427, test/loss=1.29125, test/num_examples=3003, total_duration=33054.1, train/accuracy=0.700828, train/bleu=35.9877, train/loss=1.33011, validation/accuracy=0.690469, validation/bleu=30.4082, validation/loss=1.39926, validation/num_examples=3000 +I0917 00:13:31.731937 140257425078016 logging_writer.py:48] [155000] global_step=155000, grad_norm=0.340731, loss=1.46558 +I0917 00:13:31.734914 140285881828544 submission.py:307] 155000) loss = 1.466, grad_norm = 0.341 +I0917 00:14:48.411120 140257433470720 logging_writer.py:48] [155500] global_step=155500, grad_norm=0.374022, loss=1.46311 +I0917 00:14:48.414372 140285881828544 submission.py:307] 155500) loss = 1.463, grad_norm = 0.374 +I0917 00:16:05.208415 140257425078016 logging_writer.py:48] [156000] global_step=156000, grad_norm=0.402319, loss=1.4455 +I0917 00:16:05.211482 140285881828544 submission.py:307] 156000) loss = 1.445, grad_norm = 0.402 +I0917 00:17:22.043180 140257433470720 logging_writer.py:48] [156500] global_step=156500, grad_norm=0.382112, loss=1.43645 +I0917 00:17:22.046183 140285881828544 submission.py:307] 156500) loss = 1.436, grad_norm = 0.382 +I0917 00:18:38.891062 140257425078016 logging_writer.py:48] [157000] global_step=157000, grad_norm=0.384721, loss=1.44483 +I0917 00:18:38.894198 140285881828544 submission.py:307] 157000) loss = 1.445, grad_norm = 0.385 +I0917 00:19:55.699945 140257433470720 logging_writer.py:48] [157500] global_step=157500, grad_norm=0.351545, loss=1.45168 +I0917 00:19:55.703133 140285881828544 submission.py:307] 157500) loss = 1.452, grad_norm = 0.352 +I0917 00:21:12.542179 140257425078016 logging_writer.py:48] [158000] global_step=158000, grad_norm=0.357646, loss=1.50697 +I0917 00:21:12.545220 140285881828544 submission.py:307] 158000) loss = 1.507, grad_norm = 0.358 +I0917 00:22:29.346220 140257433470720 logging_writer.py:48] [158500] global_step=158500, grad_norm=0.3798, loss=1.39659 +I0917 00:22:29.349171 140285881828544 submission.py:307] 158500) loss = 1.397, grad_norm = 0.380 +I0917 00:23:36.188037 140257425078016 logging_writer.py:48] [158936] global_step=158936, preemption_count=0, score=24474.6 +I0917 00:23:36.205723 140285881828544 submission_runner.py:857] Final wmt score: 24474.575791597366