Fix tsmatchsel() to account properly for null rows.

tglsfdc · tglsfdc · commit 52b60530f257 · 2011-02-17T19:00:49.000-05:00
ts_typanalyze.c computes MCE statistics as fractions of the non-null rows,
which seems fairly reasonable, and anyway changing it in released versions
wouldn't be a good idea.  But then ts_selfuncs.c has to account for that.
Failure to do so results in overestimates in columns with a significant
fraction of null documents.  Back-patch to 8.4 where this stuff was
introduced.

Jesper Krogh
diff --git a/src/backend/tsearch/ts_selfuncs.c b/src/backend/tsearch/ts_selfuncs.c
@@ -189,11 +189,17 @@ tsquerysel(VariableStatData *vardata, Datum constval)
 			/* No most-common-elements info, so do without */
 			selec = tsquery_opr_selec_no_stats(query);
 		}
+
+		/*
+		 * MCE stats count only non-null rows, so adjust for null rows.
+		 */
+		selec *= (1.0 - stats->stanullfrac);
 	}
 	else
 	{
 		/* No stats at all, so do without */
 		selec = tsquery_opr_selec_no_stats(query);
+		/* we assume no nulls here, so no stanullfrac correction */
 	}
 
 	return selec;
diff --git a/src/include/catalog/pg_statistic.h b/src/include/catalog/pg_statistic.h
@@ -246,6 +246,8 @@ typedef FormData_pg_statistic *Form_pg_statistic;
  * type with identifiable elements (for instance, tsvector).  staop contains
  * the equality operator appropriate to the element type.  stavalues contains
  * the most common element values, and stanumbers their frequencies.  Unlike
+ * MCV slots, frequencies are measured as the fraction of non-null rows the
+ * element value appears in, not the frequency of all rows.  Also unlike
  * MCV slots, the values are sorted into order (to support binary search
  * for a particular value).  Since this puts the minimum and maximum
  * frequencies at unpredictable spots in stanumbers, there are two extra

Original file line number	Diff line number	Diff line change
`@@ -189,11 +189,17 @@ tsquerysel(VariableStatData *vardata, Datum constval)`
`189`	`189`	`/* No most-common-elements info, so do without */`
`190`	`190`	`selec = tsquery_opr_selec_no_stats(query);`
`191`	`191`	`}`
	`192`	`+`
	`193`	`+ /*`
	`194`	`+ * MCE stats count only non-null rows, so adjust for null rows.`
	`195`	`+ */`
	`196`	`+ selec *= (1.0 - stats->stanullfrac);`
`192`	`197`	`}`
`193`	`198`	`else`
`194`	`199`	`{`
`195`	`200`	`/* No stats at all, so do without */`
`196`	`201`	`selec = tsquery_opr_selec_no_stats(query);`
	`202`	`+ /* we assume no nulls here, so no stanullfrac correction */`
`197`	`203`	`}`
`198`	`204`
`199`	`205`	`return selec;`