A few fixes from safety eval running

pamelafox · pamelafox · commit 74a6c3dce69d · 2025-04-28T22:54:12.000Z
diff --git a/evals/safety_evaluation.py b/evals/safety_evaluation.py
@@ -106,15 +106,13 @@ async def run_simulator(target_url: str, max_simulations: int):
             else:
                 logger.info(f"Failing score from:\nQ: {query}\nA: {answer}\n{evaluator} score: {eval_score}")
             numeric_severity_score = eval_score[f"{evaluator}_score"]
-            if isinstance(numeric_severity_score, float):
+            if isinstance(numeric_severity_score, float) or isinstance(numeric_severity_score, int):
                 summary_scores[evaluator]["score_total"] += numeric_severity_score
 
     # Compute the overall statistics
     for evaluator in evaluators:
         if len(outputs) > 0:
-            summary_scores[evaluator]["mean_score"] = (
-                summary_scores[evaluator]["score_total"] / summary_scores[evaluator]["low_count"]
-            )
+            summary_scores[evaluator]["mean_score"] = summary_scores[evaluator]["score_total"] / len(outputs)
             summary_scores[evaluator]["low_rate"] = summary_scores[evaluator]["low_count"] / len(outputs)
 
     # Save summary scores
diff --git a/src/backend/fastapi_app/postgres_models.py b/src/backend/fastapi_app/postgres_models.py
@@ -52,7 +52,7 @@ def to_str_for_embedding(self):
 table_name = Item.__tablename__
 
 index_ada002 = Index(
-    "hnsw_index_for_cosine_{table_name}_embedding_ada002",
+    f"hnsw_index_for_cosine_{table_name}_embedding_ada002",
     Item.embedding_ada002,
     postgresql_using="hnsw",
     postgresql_with={"m": 16, "ef_construction": 64},
diff --git a/src/backend/fastapi_app/query_rewriter.py b/src/backend/fastapi_app/query_rewriter.py
@@ -69,7 +69,7 @@ def extract_search_arguments(original_user_query: str, chat_completion: ChatComp
                 arg = json.loads(function.arguments)
                 # Even though its required, search_query is not always specified
                 search_query = arg.get("search_query", original_user_query)
-                if "price_filter" in arg and arg["price_filter"]:
+                if "price_filter" in arg and arg["price_filter"] and isinstance(arg["price_filter"], dict):
                     price_filter = arg["price_filter"]
                     filters.append(
                         {
@@ -78,7 +78,7 @@ def extract_search_arguments(original_user_query: str, chat_completion: ChatComp
                             "value": price_filter["value"],
                         }
                     )
-                if "brand_filter" in arg and arg["brand_filter"]:
+                if "brand_filter" in arg and arg["brand_filter"] and isinstance(arg["brand_filter"], dict):
                     brand_filter = arg["brand_filter"]
                     filters.append(
                         {
diff --git a/src/backend/fastapi_app/routes/api_routes.py b/src/backend/fastapi_app/routes/api_routes.py
@@ -145,6 +145,7 @@ async def chat_handler(
         if isinstance(e, APIError) and e.code == "content_filter":
             return ERROR_FILTER
         else:
+            logging.exception("Exception while generating response: %s", e)
             return {"error": str(e)}
 
 

Original file line number	Diff line number	Diff line change
`@@ -69,7 +69,7 @@ def extract_search_arguments(original_user_query: str, chat_completion: ChatComp`
`69`	`69`	`arg = json.loads(function.arguments)`
`70`	`70`	`# Even though its required, search_query is not always specified`
`71`	`71`	`search_query = arg.get("search_query", original_user_query)`
`72`		`- if "price_filter" in arg and arg["price_filter"]:`
	`72`	`+ if "price_filter" in arg and arg["price_filter"] and isinstance(arg["price_filter"], dict):`
`73`	`73`	`price_filter = arg["price_filter"]`
`74`	`74`	`filters.append(`
`75`	`75`	`{`
`@@ -78,7 +78,7 @@ def extract_search_arguments(original_user_query: str, chat_completion: ChatComp`
`78`	`78`	`"value": price_filter["value"],`
`79`	`79`	`}`
`80`	`80`	`)`
`81`		`- if "brand_filter" in arg and arg["brand_filter"]:`
	`81`	`+ if "brand_filter" in arg and arg["brand_filter"] and isinstance(arg["brand_filter"], dict):`
`82`	`82`	`brand_filter = arg["brand_filter"]`
`83`	`83`	`filters.append(`
`84`	`84`	`{`