Multiple paper open-source codes of the Microsoft Research Asia DKI group
Training data and LogiGAN pretrained model. The warmup pre-traing data is gen_train_A.json, and the continual adversarial pre-training data is gen_train_B.json (gen_train_B.json with elastic searched pseudo samples is stored in train_data_with_es/gen_train_src_es.jsonl).
LEMON dataset and pre-training corpus.
{
"ROUGE": 0.8954699040374693,
"_ROUGE1": 0.9248370079585566,
"_ROUGE2": 0.8548729804396925,
"EM": 0.4933385579937304,
"_P1": 0.7443478260869565,
"_R1": 0.6512335615693946,
"F1": 0.694684366123703,
"_P2": 0.6040515653775322,
"_R2": 0.5369713506139154,
"F2": 0.5685396504405605,
"_P3": 0.515867089789061,
"_R3": 0.4619306310071041,
"F3": 0.48741126151946734,
"_BLEU1": 0.9203424772022362,
"_BLEU2": 0.8919446800461631,
"_BLEU3": 0.8644065076063657,
"BLEU4": 0.836555297206264,
"loss": 0.012869063752786444
}
{
"ROUGE": 0.8761914250271031,
"_ROUGE1": 0.9096109449987978,
"_ROUGE2": 0.8299625766685176,
"EM": 0.43260188087774293,
"_P1": 0.663828689370485,
"_R1": 0.5592870340180415,
"F1": 0.6070901905267504,
"_P2": 0.5118311660626732,
"_R2": 0.43674397453387903,
"F2": 0.4713156990724837,
"_P3": 0.42003325833903943,
"_R3": 0.3588800668616799,
"F3": 0.38705606634216694,
"_BLEU1": 0.9049646708813686,
"_BLEU2": 0.8726836197111325,
"_BLEU3": 0.8417857314177102,
"BLEU4": 0.810576724115273,
"loss": 0.01269258979597015
}
{
"ROUGE": 0.7672702010110017,
"_ROUGE1": 0.8060129493808116,
"_ROUGE2": 0.6272019845050271,
"EM": 0.1825525040387722,
"_P1": 0.6173277359583093,
"_R1": 0.34435562015503873,
"F1": 0.44210035247771096,
"_P2": 0.39885553659271156,
"_R2": 0.24790964682391115,
"F2": 0.3057682687497595,
"_P3": 0.31307374374592123,
"_R3": 0.20793481087641227,
"F3": 0.24989581886373105,
"_BLEU1": 0.7020165933536937,
"_BLEU2": 0.6217319698491652,
"_BLEU3": 0.5573085608208992,
"BLEU4": 0.4977857601746373,
"loss": 0.01892541069261615
}
{
"ROUGE": 0.8927719935645371,
"_ROUGE1": 0.9235141523874184,
"_ROUGE2": 0.8481181915297369,
"EM": 0.536,
"_P1": 0.8750417083750417,
"_R1": 0.7598145733738955,
"F1": 0.8133674497945259,
"_P2": 0.7759128386336867,
"_R2": 0.6749487704918032,
"F2": 0.7219178082191782,
"_P3": 0.7060550971892043,
"_R3": 0.6137109517442567,
"F3": 0.6566523605150214,
"_BLEU1": 0.8953007499145113,
"_BLEU2": 0.8627271154439679,
"_BLEU3": 0.8298150265279266,
"BLEU4": 0.7961327186223426,
"loss": 0.07524961760888497
}
{
"ROUGE": 0.9394040084189113,
"_ROUGE1": 0.961865057419486,
"_ROUGE2": 0.9113051224617216,
"EM": 0.688,
"_P1": 0.9451903332806824,
"_R1": 0.8668694770389685,
"F1": 0.9043373129817137,
"_P2": 0.8648273949812838,
"_R2": 0.7989241803278688,
"F2": 0.8305705345849144,
"_P3": 0.8075098814229249,
"_R3": 0.7449860216360763,
"F3": 0.774988935954985,
"_BLEU1": 0.9405510823944796,
"_BLEU2": 0.9172718486250105,
"_BLEU3": 0.8932687251641028,
"BLEU4": 0.8691863201601382,
"loss": 0.2084200546145439
}
{
"best_epoch": 35,
"peak_cpu_memory_MB": 0,
"peak_gpu_0_memory_MB": 1,
"peak_gpu_1_memory_MB": 22811,
"peak_gpu_2_memory_MB": 22671,
"peak_gpu_3_memory_MB": 19426,
"peak_gpu_4_memory_MB": 22798,
"peak_gpu_5_memory_MB": 17286,
"peak_gpu_6_memory_MB": 21951,
"peak_gpu_7_memory_MB": 1,
"training_duration": "12:23:41.284435",
"training_start_epoch": 0,
"training_epochs": 44,
"epoch": 44,
"training_action_exact_match": 0,
"training__action_inter_exact_match": 0,
"training__action_turn_1_exact_match": 0,
"training__action_turn_2_exact_match": 0,
"training__action_turn_3_exact_match": 0,
"training__action_turn_4_exact_match": 0,
"training_sql_exact_match": 0,
"training__sql_inter_exact_match": 0,
"training__sql_turn_1_exact_match": 0,
"training__sql_turn_2_exact_match": 0,
"training__sql_turn_3_exact_match": 0,
"training__sql_turn_4_exact_match": 0,
"training__copy": 0,
"training_info": 0,
"training_loss": 0.17617646926255137,
"training_cpu_memory_MB": 0.0,
"training_gpu_0_memory_MB": 1,
"training_gpu_1_memory_MB": 3,
"training_gpu_2_memory_MB": 1743,
"training_gpu_3_memory_MB": 19426,
"training_gpu_4_memory_MB": 1981,
"training_gpu_5_memory_MB": 16566,
"training_gpu_6_memory_MB": 19858,
"training_gpu_7_memory_MB": 1,
"validation_action_exact_match": 0.30883813306852037,
"validation__action_inter_exact_match": 0.07849829351535836,
"validation__action_turn_1_exact_match": 0.3822525597269625,
"validation__action_turn_2_exact_match": 0.2771929824561403,
"validation__action_turn_3_exact_match": 0.28688524590163933,
"validation__action_turn_4_exact_match": 0.2702702702702703,
"validation_sql_exact_match": 0.3118172790466733,
"validation__sql_inter_exact_match": 0.08532423208191127,
"validation__sql_turn_1_exact_match": 0.3993174061433447,
"validation__sql_turn_2_exact_match": 0.2807017543859649,
"validation__sql_turn_3_exact_match": 0.29098360655737704,
"validation__sql_turn_4_exact_match": 0.24864864864864866,
"validation__copy": 0,
"validation_info": 0,
"validation_loss": 0.42742106318473816,
"best_validation_action_exact_match": 0.323733862959285,
"best_validation__action_inter_exact_match": 0.08191126279863481,
"best_validation__action_turn_1_exact_match": 0.3856655290102389,
"best_validation__action_turn_2_exact_match": 0.3017543859649123,
"best_validation__action_turn_3_exact_match": 0.2786885245901639,
"best_validation__action_turn_4_exact_match": 0.31891891891891894,
"best_validation_sql_exact_match": 0.3227408142999007,
"best_validation__sql_inter_exact_match": 0.08532423208191127,
"best_validation__sql_turn_1_exact_match": 0.40273037542662116,
"best_validation__sql_turn_2_exact_match": 0.2912280701754386,
"best_validation__sql_turn_3_exact_match": 0.2786885245901639,
"best_validation__sql_turn_4_exact_match": 0.3027027027027027,
"best_validation__copy": 0,
"best_validation_info": 0,
"best_validation_loss": 0.40968313813209534
}
{
"best_epoch": 24,
"peak_cpu_memory_MB": 0,
"peak_gpu_0_memory_MB": 1,
"peak_gpu_1_memory_MB": 22815,
"peak_gpu_2_memory_MB": 22671,
"peak_gpu_3_memory_MB": 19422,
"peak_gpu_4_memory_MB": 22974,
"peak_gpu_5_memory_MB": 17284,
"peak_gpu_6_memory_MB": 21951,
"peak_gpu_7_memory_MB": 1,
"training_duration": "6:32:27.738503",
"training_start_epoch": 0,
"training_epochs": 33,
"epoch": 33,
"training_action_exact_match": 0,
"training__action_inter_exact_match": 0,
"training__action_turn_1_exact_match": 0,
"training__action_turn_2_exact_match": 0,
"training__action_turn_3_exact_match": 0,
"training__action_turn_4_exact_match": 0,
"training_sql_exact_match": 0,
"training__sql_inter_exact_match": 0,
"training__sql_turn_1_exact_match": 0,
"training__sql_turn_2_exact_match": 0,
"training__sql_turn_3_exact_match": 0,
"training__sql_turn_4_exact_match": 0,
"training_loss": 0.18920585506216245,
"training_cpu_memory_MB": 0.0,
"training_gpu_0_memory_MB": 1,
"training_gpu_1_memory_MB": 22237,
"training_gpu_2_memory_MB": 19158,
"training_gpu_3_memory_MB": 19422,
"training_gpu_4_memory_MB": 20185,
"training_gpu_5_memory_MB": 17284,
"training_gpu_6_memory_MB": 21951,
"training_gpu_7_memory_MB": 1,
"validation_action_exact_match": 0.29890764647467727,
"validation__action_inter_exact_match": 0.07167235494880546,
"validation__action_turn_1_exact_match": 0.36860068259385664,
"validation__action_turn_2_exact_match": 0.2771929824561403,
"validation__action_turn_3_exact_match": 0.26229508196721313,
"validation__action_turn_4_exact_match": 0.2702702702702703,
"validation_sql_exact_match": 0.29791459781529295,
"validation__sql_inter_exact_match": 0.08191126279863481,
"validation__sql_turn_1_exact_match": 0.378839590443686,
"validation__sql_turn_2_exact_match": 0.2736842105263158,
"validation__sql_turn_3_exact_match": 0.26229508196721313,
"validation__sql_turn_4_exact_match": 0.25405405405405407,
"validation_loss": 0.47665515542030334,
"best_validation_action_exact_match": 0.32075471698113206,
"best_validation__action_inter_exact_match": 0.09215017064846416,
"best_validation__action_turn_1_exact_match": 0.3890784982935154,
"best_validation__action_turn_2_exact_match": 0.2912280701754386,
"best_validation__action_turn_3_exact_match": 0.29098360655737704,
"best_validation__action_turn_4_exact_match": 0.2972972972972973,
"best_validation_sql_exact_match": 0.3187686196623635,
"best_validation__sql_inter_exact_match": 0.09897610921501707,
"best_validation__sql_turn_1_exact_match": 0.3993174061433447,
"best_validation__sql_turn_2_exact_match": 0.28421052631578947,
"best_validation__sql_turn_3_exact_match": 0.28688524590163933,
"best_validation__sql_turn_4_exact_match": 0.2864864864864865,
"best_validation_loss": 0.45512670278549194
}
{
"best_epoch": 23,
"peak_cpu_memory_MB": 0,
"peak_gpu_0_memory_MB": 1,
"peak_gpu_1_memory_MB": 3489,
"peak_gpu_2_memory_MB": 16884,
"peak_gpu_3_memory_MB": 16857,
"peak_gpu_4_memory_MB": 16886,
"peak_gpu_5_memory_MB": 16816,
"peak_gpu_6_memory_MB": 20163,
"peak_gpu_7_memory_MB": 19975,
"training_duration": "8:04:29.261273",
"training_start_epoch": 0,
"training_epochs": 32,
"epoch": 32,
"training_action_exact_match": 0,
"training__action_inter_exact_match": 0,
"training__action_turn_1_exact_match": 0,
"training__action_turn_2_exact_match": 0,
"training__action_turn_3_exact_match": 0,
"training__action_turn_4_exact_match": 0,
"training_sql_exact_match": 0,
"training__sql_inter_exact_match": 0,
"training__sql_turn_1_exact_match": 0,
"training__sql_turn_2_exact_match": 0,
"training__sql_turn_3_exact_match": 0,
"training__sql_turn_4_exact_match": 0,
"training_loss": 0.05288350063624136,
"training_cpu_memory_MB": 0.0,
"training_gpu_0_memory_MB": 1,
"training_gpu_1_memory_MB": 1751,
"training_gpu_2_memory_MB": 16884,
"training_gpu_3_memory_MB": 16857,
"training_gpu_4_memory_MB": 16886,
"training_gpu_5_memory_MB": 16816,
"training_gpu_6_memory_MB": 20163,
"training_gpu_7_memory_MB": 2,
"validation_action_exact_match": 0.3843098311817279,
"validation__action_inter_exact_match": 0.12286689419795221,
"validation__action_turn_1_exact_match": 0.48464163822525597,
"validation__action_turn_2_exact_match": 0.35789473684210527,
"validation__action_turn_3_exact_match": 0.3483606557377049,
"validation__action_turn_4_exact_match": 0.31351351351351353,
"validation_sql_exact_match": 0.38828202581926513,
"validation__sql_inter_exact_match": 0.1296928327645051,
"validation__sql_turn_1_exact_match": 0.49829351535836175,
"validation__sql_turn_2_exact_match": 0.3473684210526316,
"validation__sql_turn_3_exact_match": 0.36065573770491804,
"validation__sql_turn_4_exact_match": 0.31351351351351353,
"validation_loss": 0.24153371155261993,
"best_validation_action_exact_match": 0.38927507447864945,
"best_validation__action_inter_exact_match": 0.13651877133105803,
"best_validation__action_turn_1_exact_match": 0.47440273037542663,
"best_validation__action_turn_2_exact_match": 0.38596491228070173,
"best_validation__action_turn_3_exact_match": 0.3442622950819672,
"best_validation__action_turn_4_exact_match": 0.31891891891891894,
"best_validation_sql_exact_match": 0.39225422045680236,
"best_validation__sql_inter_exact_match": 0.14334470989761092,
"best_validation__sql_turn_1_exact_match": 0.4948805460750853,
"best_validation__sql_turn_2_exact_match": 0.3684210526315789,
"best_validation__sql_turn_3_exact_match": 0.36475409836065575,
"best_validation__sql_turn_4_exact_match": 0.3027027027027027,
"best_validation_loss": 0.20682337880134583
}