DeepSeek/ESFT
1
0
Fork 0
mirror of https://github.com/deepseek-ai/ESFT synced 2024-11-21 19:17:39 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Update benchmarks.py

Browse Source
This commit is contained in:
Zihan Wang 2024-07-11 11:33:47 +08:00 committed by GitHub
parent 38c8074be0
commit 26b4fc4a8a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
benchmarks.py
View File

@ -138,7 +138,7 @@ class IntentEvaluator(BaseEvaluator):
prediction = json.loads(prediction) prediction = json.loads(prediction)
except: except:
print(f"unable to parse prediction {prediction} of example with gt {ground_truth}") print(f"unable to parse prediction {prediction} of example with gt {ground_truth}")
return {'intent_acc': 0, 'slots_f1': 0, 'em': 0} return 0.0
intent_em = prediction.get('intent', '') == ground_truth.get('intent', '') intent_em = prediction.get('intent', '') == ground_truth.get('intent', '')
@ -147,7 +147,7 @@ class IntentEvaluator(BaseEvaluator):
pred_slots = {(k, str(tuple(sorted([str(i).replace(" ", "") for i in v]))) if isinstance(v, list) else v.replace(" ", "")) for k, v in prediction.get('slots', {}).items()} pred_slots = {(k, str(tuple(sorted([str(i).replace(" ", "") for i in v]))) if isinstance(v, list) else v.replace(" ", "")) for k, v in prediction.get('slots', {}).items()}
except: except:
print(f"OK to parse prediction slots {prediction} of example with gt {ground_truth}, but failed in processing the contents.") print(f"OK to parse prediction slots {prediction} of example with gt {ground_truth}, but failed in processing the contents.")
return {'intent_acc': 0, 'slots_f1': 0, 'em': 0} return 0.0
correct_slots = pred_slots.intersection(gt_slots) correct_slots = pred_slots.intersection(gt_slots)
slots_em = (len(correct_slots) == len(pred_slots)) and (len(correct_slots) == len(gt_slots)) slots_em = (len(correct_slots) == len(pred_slots)) and (len(correct_slots) == len(gt_slots))