Skip to content

Commit 3d7eee2

Browse files
danolivodanolivo
committed
Add reliability factor (rfactor) into interface of learning procedures.
1 parent 0dc6595 commit 3d7eee2

File tree

10 files changed

+225
-226
lines changed

10 files changed

+225
-226
lines changed

‎aqo.c

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -71,21 +71,6 @@ int auto_tuning_infinite_loop = 8;
7171

7272
/* Machine learning parameters */
7373

74-
/*
75-
* Defines where we do not perform learning procedure
76-
*/
77-
const double object_selection_prediction_threshold = 0.3;
78-
79-
/*
80-
* This parameter tell us that the new learning sample object has very small
81-
* distance from one whose features stored in matrix already.
82-
* In this case we will not to add new line in matrix, but will modify this
83-
* nearest neighbor features and cardinality with linear smoothing by
84-
* learning_rate coefficient.
85-
*/
86-
const double object_selection_threshold = 0.1;
87-
const double learning_rate = 1e-1;
88-
8974
/* The number of nearest neighbors which will be chosen for ML-operations */
9075
int aqo_k = 3;
9176
double log_selectivity_lower_bound = -30;

‎aqo.h

Lines changed: 7 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,7 @@
144144
#include "utils/fmgroids.h"
145145
#include "utils/snapmgr.h"
146146

147+
#include "machine_learning.h"
147148

148149
/* Check PostgreSQL version (9.6.0 contains important changes in planner) */
149150
#if PG_VERSION_NUM < 90600
@@ -237,12 +238,6 @@ extern double auto_tuning_convergence_error;
237238

238239
/* Machine learning parameters */
239240

240-
/* Max number of matrix rows - max number of possible neighbors. */
241-
#define aqo_K (30)
242-
243-
extern const double object_selection_prediction_threshold;
244-
extern const double object_selection_threshold;
245-
extern const double learning_rate;
246241
extern int aqo_k;
247242
extern double log_selectivity_lower_bound;
248243

@@ -285,17 +280,13 @@ extern bool find_query(uint64 qhash, QueryContextData *ctx);
285280
extern bool update_query(uint64 qhash, uint64 fhash,
286281
bool learn_aqo, bool use_aqo, bool auto_tuning);
287282
extern bool add_query_text(uint64 query_hash, const char *query_string);
288-
extern bool load_fss_ext(uint64 fs, int fss,
289-
int ncols, double **matrix, double *targets, int *rows,
283+
extern bool load_fss_ext(uint64 fs, int fss, OkNNrdata *data,
290284
List **relids, bool isSafe);
291-
extern bool load_fss(uint64 fhash, int fss_hash,
292-
int ncols, double **matrix, double *targets, int *rows,
293-
List **relids);
294-
extern bool update_fss_ext(uint64 fhash, int fsshash, int nrows, int ncols,
295-
double **matrix, double *targets, List *relids,
296-
bool isTimedOut);
297-
extern bool update_fss(uint64 fhash, int fss_hash, int nrows, int ncols,
298-
double **matrix, double *targets, List *relids);
285+
extern bool load_fss(uint64 fhash, int fss_hash, OkNNrdata *data, List **relids);
286+
extern bool update_fss_ext(uint64 fhash, int fsshash, OkNNrdata *data,
287+
List *relids, bool isTimedOut);
288+
extern bool update_fss(uint64 fhash, int fss_hash, OkNNrdata *data,
289+
List *relids);
299290
QueryStat *get_aqo_stat(uint64 query_hash);
300291
void update_aqo_stat(uint64 query_hash, QueryStat * stat);
301292
extern bool my_index_insert(Relation indexRelation, Datum *values, bool *isnull,
@@ -324,14 +315,6 @@ void aqo_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction,
324315
uint64 count, bool execute_once);
325316
void aqo_ExecutorEnd(QueryDesc *queryDesc);
326317

327-
/* Machine learning techniques */
328-
extern double OkNNr_predict(int nrows, int ncols,
329-
double **matrix, const double *targets,
330-
double *features);
331-
extern int OkNNr_learn(int matrix_rows, int matrix_cols,
332-
double **matrix, double *targets,
333-
double *features, double target);
334-
335318
/* Automatic query tuning */
336319
extern void automatical_query_tuning(uint64 query_hash, QueryStat * stat);
337320

‎cardinality_estimation.c

Lines changed: 14 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
#include "aqo.h"
2323
#include "hash.h"
24+
#include "machine_learning.h"
2425

2526
#ifdef AQO_DEBUG_PRINT
2627
static void
@@ -59,15 +60,12 @@ predict_debug_output(List *clauses, List *selectivities,
5960
*/
6061
double
6162
predict_for_relation(List *clauses, List *selectivities,
62-
List *relids, int *fss_hash)
63+
List *relids, int *fss)
6364
{
64-
int nfeatures;
65-
double *matrix[aqo_K];
66-
double targets[aqo_K];
67-
double *features;
68-
double result;
69-
int rows;
70-
int i;
65+
double *features;
66+
double result;
67+
int i;
68+
OkNNrdata data;
7169

7270
if (relids == NIL)
7371
/*
@@ -76,16 +74,15 @@ predict_for_relation(List *clauses, List *selectivities,
7674
*/
7775
return -4.;
7876

79-
*fss_hash = get_fss_for_object(relids, clauses,
80-
selectivities, &nfeatures, &features);
77+
*fss = get_fss_for_object(relids, clauses,
78+
selectivities, &data.cols, &features);
8179

82-
if (nfeatures > 0)
80+
if (data.cols > 0)
8381
for (i = 0; i < aqo_K; ++i)
84-
matrix[i] = palloc0(sizeof(**matrix) * nfeatures);
82+
data.matrix[i] = palloc0(sizeof(double) * data.cols);
8583

86-
if (load_fss_ext(query_context.fspace_hash, *fss_hash, nfeatures, matrix,
87-
targets, &rows, NULL, true))
88-
result = OkNNr_predict(rows, nfeatures, matrix, targets, features);
84+
if (load_fss_ext(query_context.fspace_hash, *fss, &data, NULL, true))
85+
result = OkNNr_predict(&data, features);
8986
else
9087
{
9188
/*
@@ -100,10 +97,10 @@ predict_for_relation(List *clauses, List *selectivities,
10097
predict_debug_output(clauses, selectivities, relids, *fss_hash, result);
10198
#endif
10299
pfree(features);
103-
if (nfeatures > 0)
100+
if (data.cols > 0)
104101
{
105102
for (i = 0; i < aqo_K; ++i)
106-
pfree(matrix[i]);
103+
pfree(data.matrix[i]);
107104
}
108105

109106
if (result < 0)

‎cardinality_hooks.c

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
#include "aqo.h"
3131
#include "cardinality_hooks.h"
3232
#include "hash.h"
33+
#include "machine_learning.h"
3334
#include "path_utils.h"
3435

3536
estimate_num_groups_hook_type prev_estimate_num_groups_hook = NULL;
@@ -137,12 +138,12 @@ default_estimate_num_groups(PlannerInfo *root, List *groupExprs,
137138
void
138139
aqo_set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel)
139140
{
140-
double predicted;
141-
Oid relid;
142-
List *relids = NIL;
143-
List *selectivities = NULL;
144-
List *clauses;
145-
int fss = 0;
141+
double predicted;
142+
Oid relid;
143+
List *relids = NIL;
144+
List *selectivities = NULL;
145+
List *clauses;
146+
int fss = 0;
146147

147148
if (IsQueryDisabled())
148149
/* Fast path. */
@@ -411,10 +412,9 @@ static double
411412
predict_num_groups(PlannerInfo *root, Path *subpath, List *group_exprs,
412413
int *fss)
413414
{
414-
int child_fss = 0;
415-
double prediction;
416-
int rows;
417-
double target;
415+
int child_fss = 0;
416+
double prediction;
417+
OkNNrdata data;
418418

419419
if (subpath->parent->predicted_cardinality > 0.)
420420
/* A fast path. Here we can use a fss hash of a leaf. */
@@ -431,13 +431,13 @@ predict_num_groups(PlannerInfo *root, Path *subpath, List *group_exprs,
431431
}
432432

433433
*fss = get_grouped_exprs_hash(child_fss, group_exprs);
434+
memset(&data, 0, sizeof(OkNNrdata));
434435

435-
if (!load_fss_ext(query_context.fspace_hash, *fss, 0, NULL,
436-
&target, &rows, NULL, true))
436+
if (!load_fss_ext(query_context.fspace_hash, *fss, &data, NULL, true))
437437
return -1;
438438

439-
Assert(rows == 1);
440-
prediction = exp(target);
439+
Assert(data.rows == 1);
440+
prediction = exp(data.targets[0]);
441441
return (prediction <= 0) ? -1 : prediction;
442442
}
443443

‎learn_cache.c

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,7 @@ lc_init(void)
6060
}
6161

6262
bool
63-
lc_update_fss(uint64 fs, int fss, int nrows, int ncols,
64-
double **matrix, double *targets, List *relids)
63+
lc_update_fss(uint64 fs, int fss, OkNNrdata *data, List *relids)
6564
{
6665
htab_key key = {fs, fss};
6766
htab_entry *entry;
@@ -81,15 +80,15 @@ lc_update_fss(uint64 fs, int fss, int nrows, int ncols,
8180
list_free(entry->relids);
8281
}
8382

84-
entry->nrows = nrows;
85-
entry->ncols = ncols;
83+
entry->nrows = data->rows;
84+
entry->ncols = data->cols;
8685
for (i = 0; i < entry->nrows; ++i)
8786
{
88-
entry->matrix[i] = palloc(sizeof(double) * ncols);
89-
memcpy(entry->matrix[i], matrix[i], sizeof(double) * ncols);
87+
entry->matrix[i] = palloc(sizeof(double) * data->cols);
88+
memcpy(entry->matrix[i], data->matrix[i], sizeof(double) * data->cols);
9089
}
91-
entry->targets = palloc(sizeof(double) * nrows);
92-
memcpy(entry->targets, targets, sizeof(double) * nrows);
90+
entry->targets = palloc(sizeof(double) * data->rows);
91+
memcpy(entry->targets, data->targets, sizeof(double) * data->rows);
9392
entry->relids = list_copy(relids);
9493

9594
MemoryContextSwitchTo(memctx);
@@ -116,8 +115,7 @@ lc_has_fss(uint64 fs, int fss)
116115
* XXX That to do with learning tails, living in the cache?
117116
*/
118117
bool
119-
lc_load_fss(uint64 fs, int fss, int ncols, double **matrix,
120-
double *targets, int *nrows, List **relids)
118+
lc_load_fss(uint64 fs, int fss, OkNNrdata *data, List **relids)
121119
{
122120
htab_key key = {fs, fss};
123121
htab_entry *entry;
@@ -134,11 +132,11 @@ lc_load_fss(uint64 fs, int fss, int ncols, double **matrix,
134132
elog(NOTICE, "[AQO] Load ML data for fs %lu, fss %d from the cache",
135133
fs, fss);
136134

137-
*nrows = entry->nrows;
138-
Assert(entry->ncols == ncols);
135+
data->rows = entry->nrows;
136+
Assert(entry->ncols == data->cols);
139137
for (i = 0; i < entry->nrows; ++i)
140-
memcpy(matrix[i], entry->matrix[i], sizeof(double) * ncols);
141-
memcpy(targets, entry->targets, sizeof(double) * entry->nrows);
138+
memcpy(data->matrix[i], entry->matrix[i], sizeof(double) * data->cols);
139+
memcpy(data->targets, entry->targets, sizeof(double) * entry->nrows);
142140
if (relids)
143141
*relids = list_copy(entry->relids);
144142
return true;

‎learn_cache.h

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,16 @@
33

44
#include "nodes/pg_list.h"
55

6+
#include "machine_learning.h"
7+
68
extern bool aqo_learn_statement_timeout;
79

810
extern void lc_init(void);
9-
extern bool lc_update_fss(uint64 fhash, int fsshash, int nrows, int ncols,
10-
double **matrix, double *targets, List *relids);
11+
extern bool lc_update_fss(uint64 fhash, int fsshash, OkNNrdata *data,
12+
List *relids);
1113
extern bool lc_has_fss(uint64 fhash, int fss);
12-
extern bool lc_load_fss(uint64 fhash, int fsshash, int ncols,
13-
double **matrix, double *targets, int *nrows,
14-
List **relids);
14+
extern bool lc_load_fss(uint64 fhash, int fsshash, OkNNrdata *data,
15+
List **relids);
1516
extern void lc_remove_fss(uint64 fhash, int fss_hash);
1617
extern void lc_assign_hook(bool newval, void *extra);
1718

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy