From 911e70207703799605f5a0e8aad9f06cff067c63 Mon Sep 17 00:00:00 2001 From: Alexander Korotkov Date: Mon, 30 Mar 2020 19:17:11 +0300 Subject: Implement operator class parameters PostgreSQL provides set of template index access methods, where opclasses have much freedom in the semantics of indexing. These index AMs are GiST, GIN, SP-GiST and BRIN. There opclasses define representation of keys, operations on them and supported search strategies. So, it's natural that opclasses may be faced some tradeoffs, which require user-side decision. This commit implements opclass parameters allowing users to set some values, which tell opclass how to index the particular dataset. This commit doesn't introduce new storage in system catalog. Instead it uses pg_attribute.attoptions, which is used for table column storage options but unused for index attributes. In order to evade changing signature of each opclass support function, we implement unified way to pass options to opclass support functions. Options are set to fn_expr as the constant bytea expression. It's possible due to the fact that opclass support functions are executed outside of expressions, so fn_expr is unused for them. This commit comes with some examples of opclass options usage. We parametrize signature length in GiST. That applies to multiple opclasses: tsvector_ops, gist__intbig_ops, gist_ltree_ops, gist__ltree_ops, gist_trgm_ops and gist_hstore_ops. Also we parametrize maximum number of integer ranges for gist__int_ops. However, the main future usage of this feature is expected to be json, where users would be able to specify which way to index particular json parts. Catversion is bumped. Discussion: https://postgr.es/m/d22c3a18-31c7-1879-fc11-4c1ce2f5e5af%40postgrespro.ru Author: Nikita Glukhov, revised by me Reviwed-by: Nikolay Shaplov, Robert Haas, Tom Lane, Tomas Vondra, Alvaro Herrera --- contrib/intarray/_intbig_gist.c | 189 +++++++++++++++++++++------------------- 1 file changed, 98 insertions(+), 91 deletions(-) (limited to 'contrib/intarray/_intbig_gist.c') diff --git a/contrib/intarray/_intbig_gist.c b/contrib/intarray/_intbig_gist.c index be51dac1fa7..67c44e99a9a 100644 --- a/contrib/intarray/_intbig_gist.c +++ b/contrib/intarray/_intbig_gist.c @@ -5,6 +5,7 @@ #include "_int.h" #include "access/gist.h" +#include "access/reloptions.h" #include "access/stratnum.h" #include "port/pg_bitutils.h" @@ -19,6 +20,8 @@ PG_FUNCTION_INFO_V1(g_intbig_penalty); PG_FUNCTION_INFO_V1(g_intbig_picksplit); PG_FUNCTION_INFO_V1(g_intbig_union); PG_FUNCTION_INFO_V1(g_intbig_same); +PG_FUNCTION_INFO_V1(g_intbig_options); + PG_FUNCTION_INFO_V1(_intbig_in); PG_FUNCTION_INFO_V1(_intbig_out); @@ -40,12 +43,33 @@ _intbig_out(PG_FUNCTION_ARGS) PG_RETURN_DATUM(0); } +static GISTTYPE * +_intbig_alloc(bool allistrue, int siglen, BITVECP sign) +{ + int flag = allistrue ? ALLISTRUE : 0; + int size = CALCGTSIZE(flag, siglen); + GISTTYPE *res = (GISTTYPE *) palloc(size); + + SET_VARSIZE(res, size); + res->flag = flag; + + if (!allistrue) + { + if (sign) + memcpy(GETSIGN(res), sign, siglen); + else + memset(GETSIGN(res), 0, siglen); + } + + return res; +} + /********************************************************************* ** intbig functions *********************************************************************/ static bool -_intbig_overlap(GISTTYPE *a, ArrayType *b) +_intbig_overlap(GISTTYPE *a, ArrayType *b, int siglen) { int num = ARRNELEMS(b); int32 *ptr = ARRPTR(b); @@ -54,7 +78,7 @@ _intbig_overlap(GISTTYPE *a, ArrayType *b) while (num--) { - if (GETBIT(GETSIGN(a), HASHVAL(*ptr))) + if (GETBIT(GETSIGN(a), HASHVAL(*ptr, siglen))) return true; ptr++; } @@ -63,7 +87,7 @@ _intbig_overlap(GISTTYPE *a, ArrayType *b) } static bool -_intbig_contains(GISTTYPE *a, ArrayType *b) +_intbig_contains(GISTTYPE *a, ArrayType *b, int siglen) { int num = ARRNELEMS(b); int32 *ptr = ARRPTR(b); @@ -72,7 +96,7 @@ _intbig_contains(GISTTYPE *a, ArrayType *b) while (num--) { - if (!GETBIT(GETSIGN(a), HASHVAL(*ptr))) + if (!GETBIT(GETSIGN(a), HASHVAL(*ptr, siglen))) return false; ptr++; } @@ -86,6 +110,7 @@ g_intbig_same(PG_FUNCTION_ARGS) GISTTYPE *a = (GISTTYPE *) PG_GETARG_POINTER(0); GISTTYPE *b = (GISTTYPE *) PG_GETARG_POINTER(1); bool *result = (bool *) PG_GETARG_POINTER(2); + int siglen = GET_SIGLEN(); if (ISALLTRUE(a) && ISALLTRUE(b)) *result = true; @@ -100,7 +125,7 @@ g_intbig_same(PG_FUNCTION_ARGS) sb = GETSIGN(b); *result = true; - LOOPBYTE + LOOPBYTE(siglen) { if (sa[i] != sb[i]) { @@ -116,6 +141,7 @@ Datum g_intbig_compress(PG_FUNCTION_ARGS) { GISTENTRY *entry = (GISTENTRY *) PG_GETARG_POINTER(0); + int siglen = GET_SIGLEN(); if (entry->leafkey) { @@ -123,7 +149,7 @@ g_intbig_compress(PG_FUNCTION_ARGS) ArrayType *in = DatumGetArrayTypeP(entry->key); int32 *ptr; int num; - GISTTYPE *res = (GISTTYPE *) palloc0(CALCGTSIZE(0)); + GISTTYPE *res = _intbig_alloc(false, siglen, NULL); CHECKARRVALID(in); if (ARRISEMPTY(in)) @@ -136,11 +162,10 @@ g_intbig_compress(PG_FUNCTION_ARGS) ptr = ARRPTR(in); num = ARRNELEMS(in); } - SET_VARSIZE(res, CALCGTSIZE(0)); while (num--) { - HASH(GETSIGN(res), *ptr); + HASH(GETSIGN(res), *ptr, siglen); ptr++; } @@ -161,16 +186,13 @@ g_intbig_compress(PG_FUNCTION_ARGS) BITVECP sign = GETSIGN(DatumGetPointer(entry->key)); GISTTYPE *res; - LOOPBYTE + LOOPBYTE(siglen) { if ((sign[i] & 0xff) != 0xff) PG_RETURN_POINTER(entry); } - res = (GISTTYPE *) palloc(CALCGTSIZE(ALLISTRUE)); - SET_VARSIZE(res, CALCGTSIZE(ALLISTRUE)); - res->flag = ALLISTRUE; - + res = _intbig_alloc(true, siglen, sign); retval = (GISTENTRY *) palloc(sizeof(GISTENTRY)); gistentryinit(*retval, PointerGetDatum(res), entry->rel, entry->page, @@ -184,19 +206,19 @@ g_intbig_compress(PG_FUNCTION_ARGS) static int32 -sizebitvec(BITVECP sign) +sizebitvec(BITVECP sign, int siglen) { - return pg_popcount(sign, SIGLEN); + return pg_popcount(sign, siglen); } static int -hemdistsign(BITVECP a, BITVECP b) +hemdistsign(BITVECP a, BITVECP b, int siglen) { int i, diff, dist = 0; - LOOPBYTE + LOOPBYTE(siglen) { diff = (unsigned char) (a[i] ^ b[i]); /* Using the popcount functions here isn't likely to win */ @@ -206,19 +228,19 @@ hemdistsign(BITVECP a, BITVECP b) } static int -hemdist(GISTTYPE *a, GISTTYPE *b) +hemdist(GISTTYPE *a, GISTTYPE *b, int siglen) { if (ISALLTRUE(a)) { if (ISALLTRUE(b)) return 0; else - return SIGLENBIT - sizebitvec(GETSIGN(b)); + return SIGLENBIT(siglen) - sizebitvec(GETSIGN(b), siglen); } else if (ISALLTRUE(b)) - return SIGLENBIT - sizebitvec(GETSIGN(a)); + return SIGLENBIT(siglen) - sizebitvec(GETSIGN(a), siglen); - return hemdistsign(GETSIGN(a), GETSIGN(b)); + return hemdistsign(GETSIGN(a), GETSIGN(b), siglen); } Datum @@ -228,14 +250,14 @@ g_intbig_decompress(PG_FUNCTION_ARGS) } static int32 -unionkey(BITVECP sbase, GISTTYPE *add) +unionkey(BITVECP sbase, GISTTYPE *add, int siglen) { int32 i; BITVECP sadd = GETSIGN(add); if (ISALLTRUE(add)) return 1; - LOOPBYTE + LOOPBYTE(siglen) sbase[i] |= sadd[i]; return 0; } @@ -245,29 +267,22 @@ g_intbig_union(PG_FUNCTION_ARGS) { GistEntryVector *entryvec = (GistEntryVector *) PG_GETARG_POINTER(0); int *size = (int *) PG_GETARG_POINTER(1); - BITVEC base; - int32 i, - len; - int32 flag = 0; - GISTTYPE *result; + int siglen = GET_SIGLEN(); + int32 i; + GISTTYPE *result = _intbig_alloc(false, siglen, NULL); + BITVECP base = GETSIGN(result); - MemSet((void *) base, 0, sizeof(BITVEC)); for (i = 0; i < entryvec->n; i++) { - if (unionkey(base, GETENTRY(entryvec, i))) + if (unionkey(base, GETENTRY(entryvec, i), siglen)) { - flag = ALLISTRUE; + result->flag |= ALLISTRUE; + SET_VARSIZE(result, CALCGTSIZE(ALLISTRUE, siglen)); break; } } - len = CALCGTSIZE(flag); - result = (GISTTYPE *) palloc(len); - SET_VARSIZE(result, len); - result->flag = flag; - if (!ISALLTRUE(result)) - memcpy((void *) GETSIGN(result), (void *) base, sizeof(BITVEC)); - *size = len; + *size = VARSIZE(result); PG_RETURN_POINTER(result); } @@ -280,8 +295,9 @@ g_intbig_penalty(PG_FUNCTION_ARGS) float *penalty = (float *) PG_GETARG_POINTER(2); GISTTYPE *origval = (GISTTYPE *) DatumGetPointer(origentry->key); GISTTYPE *newval = (GISTTYPE *) DatumGetPointer(newentry->key); + int siglen = GET_SIGLEN(); - *penalty = hemdist(origval, newval); + *penalty = hemdist(origval, newval, siglen); PG_RETURN_POINTER(penalty); } @@ -304,6 +320,7 @@ g_intbig_picksplit(PG_FUNCTION_ARGS) { GistEntryVector *entryvec = (GistEntryVector *) PG_GETARG_POINTER(0); GIST_SPLITVEC *v = (GIST_SPLITVEC *) PG_GETARG_POINTER(1); + int siglen = GET_SIGLEN(); OffsetNumber k, j; GISTTYPE *datum_l, @@ -336,7 +353,7 @@ g_intbig_picksplit(PG_FUNCTION_ARGS) _k = GETENTRY(entryvec, k); for (j = OffsetNumberNext(k); j <= maxoff; j = OffsetNumberNext(j)) { - size_waste = hemdist(_k, GETENTRY(entryvec, j)); + size_waste = hemdist(_k, GETENTRY(entryvec, j), siglen); if (size_waste > waste) { waste = size_waste; @@ -358,32 +375,10 @@ g_intbig_picksplit(PG_FUNCTION_ARGS) } /* form initial .. */ - if (ISALLTRUE(GETENTRY(entryvec, seed_1))) - { - datum_l = (GISTTYPE *) palloc(GTHDRSIZE); - SET_VARSIZE(datum_l, GTHDRSIZE); - datum_l->flag = ALLISTRUE; - } - else - { - datum_l = (GISTTYPE *) palloc(GTHDRSIZE + SIGLEN); - SET_VARSIZE(datum_l, GTHDRSIZE + SIGLEN); - datum_l->flag = 0; - memcpy((void *) GETSIGN(datum_l), (void *) GETSIGN(GETENTRY(entryvec, seed_1)), sizeof(BITVEC)); - } - if (ISALLTRUE(GETENTRY(entryvec, seed_2))) - { - datum_r = (GISTTYPE *) palloc(GTHDRSIZE); - SET_VARSIZE(datum_r, GTHDRSIZE); - datum_r->flag = ALLISTRUE; - } - else - { - datum_r = (GISTTYPE *) palloc(GTHDRSIZE + SIGLEN); - SET_VARSIZE(datum_r, GTHDRSIZE + SIGLEN); - datum_r->flag = 0; - memcpy((void *) GETSIGN(datum_r), (void *) GETSIGN(GETENTRY(entryvec, seed_2)), sizeof(BITVEC)); - } + datum_l = _intbig_alloc(ISALLTRUE(GETENTRY(entryvec, seed_1)), siglen, + GETSIGN(GETENTRY(entryvec, seed_1))); + datum_r = _intbig_alloc(ISALLTRUE(GETENTRY(entryvec, seed_2)), siglen, + GETSIGN(GETENTRY(entryvec, seed_2))); maxoff = OffsetNumberNext(maxoff); /* sort before ... */ @@ -392,8 +387,8 @@ g_intbig_picksplit(PG_FUNCTION_ARGS) { costvector[j - 1].pos = j; _j = GETENTRY(entryvec, j); - size_alpha = hemdist(datum_l, _j); - size_beta = hemdist(datum_r, _j); + size_alpha = hemdist(datum_l, _j, siglen); + size_beta = hemdist(datum_r, _j, siglen); costvector[j - 1].cost = Abs(size_alpha - size_beta); } qsort((void *) costvector, maxoff, sizeof(SPLITCOST), comparecost); @@ -417,20 +412,20 @@ g_intbig_picksplit(PG_FUNCTION_ARGS) continue; } _j = GETENTRY(entryvec, j); - size_alpha = hemdist(datum_l, _j); - size_beta = hemdist(datum_r, _j); + size_alpha = hemdist(datum_l, _j, siglen); + size_beta = hemdist(datum_r, _j, siglen); if (size_alpha < size_beta + WISH_F(v->spl_nleft, v->spl_nright, 0.00001)) { if (ISALLTRUE(datum_l) || ISALLTRUE(_j)) { if (!ISALLTRUE(datum_l)) - MemSet((void *) union_l, 0xff, sizeof(BITVEC)); + MemSet((void *) union_l, 0xff, siglen); } else { ptr = GETSIGN(_j); - LOOPBYTE + LOOPBYTE(siglen) union_l[i] |= ptr[i]; } *left++ = j; @@ -441,12 +436,12 @@ g_intbig_picksplit(PG_FUNCTION_ARGS) if (ISALLTRUE(datum_r) || ISALLTRUE(_j)) { if (!ISALLTRUE(datum_r)) - MemSet((void *) union_r, 0xff, sizeof(BITVEC)); + MemSet((void *) union_r, 0xff, siglen); } else { ptr = GETSIGN(_j); - LOOPBYTE + LOOPBYTE(siglen) union_r[i] |= ptr[i]; } *right++ = j; @@ -472,6 +467,7 @@ g_intbig_consistent(PG_FUNCTION_ARGS) /* Oid subtype = PG_GETARG_OID(3); */ bool *recheck = (bool *) PG_GETARG_POINTER(4); + int siglen = GET_SIGLEN(); bool retval; /* All cases served by this function are inexact */ @@ -484,6 +480,7 @@ g_intbig_consistent(PG_FUNCTION_ARGS) { retval = signconsistent((QUERYTYPE *) query, GETSIGN(DatumGetPointer(entry->key)), + siglen, false); PG_FREE_IF_COPY(query, 1); PG_RETURN_BOOL(retval); @@ -494,7 +491,8 @@ g_intbig_consistent(PG_FUNCTION_ARGS) switch (strategy) { case RTOverlapStrategyNumber: - retval = _intbig_overlap((GISTTYPE *) DatumGetPointer(entry->key), query); + retval = _intbig_overlap((GISTTYPE *) DatumGetPointer(entry->key), + query, siglen); break; case RTSameStrategyNumber: if (GIST_LEAF(entry)) @@ -502,22 +500,18 @@ g_intbig_consistent(PG_FUNCTION_ARGS) int i, num = ARRNELEMS(query); int32 *ptr = ARRPTR(query); - BITVEC qp; - BITVECP dq, + BITVECP dq = palloc0(siglen), de; - memset(qp, 0, sizeof(BITVEC)); - while (num--) { - HASH(qp, *ptr); + HASH(dq, *ptr, siglen); ptr++; } de = GETSIGN((GISTTYPE *) DatumGetPointer(entry->key)); - dq = qp; retval = true; - LOOPBYTE + LOOPBYTE(siglen) { if (de[i] != dq[i]) { @@ -526,13 +520,16 @@ g_intbig_consistent(PG_FUNCTION_ARGS) } } + pfree(dq); } else - retval = _intbig_contains((GISTTYPE *) DatumGetPointer(entry->key), query); + retval = _intbig_contains((GISTTYPE *) DatumGetPointer(entry->key), + query, siglen); break; case RTContainsStrategyNumber: case RTOldContainsStrategyNumber: - retval = _intbig_contains((GISTTYPE *) DatumGetPointer(entry->key), query); + retval = _intbig_contains((GISTTYPE *) DatumGetPointer(entry->key), + query, siglen); break; case RTContainedByStrategyNumber: case RTOldContainedByStrategyNumber: @@ -541,22 +538,18 @@ g_intbig_consistent(PG_FUNCTION_ARGS) int i, num = ARRNELEMS(query); int32 *ptr = ARRPTR(query); - BITVEC qp; - BITVECP dq, + BITVECP dq = palloc0(siglen), de; - memset(qp, 0, sizeof(BITVEC)); - while (num--) { - HASH(qp, *ptr); + HASH(dq, *ptr, siglen); ptr++; } de = GETSIGN((GISTTYPE *) DatumGetPointer(entry->key)); - dq = qp; retval = true; - LOOPBYTE + LOOPBYTE(siglen) { if (de[i] & ~dq[i]) { @@ -580,3 +573,17 @@ g_intbig_consistent(PG_FUNCTION_ARGS) PG_FREE_IF_COPY(query, 1); PG_RETURN_BOOL(retval); } + +Datum +g_intbig_options(PG_FUNCTION_ARGS) +{ + local_relopts *relopts = (local_relopts *) PG_GETARG_POINTER(0); + + init_local_reloptions(relopts, sizeof(GISTIntArrayBigOptions)); + add_local_int_reloption(relopts, "siglen", + "signature length in bytes", + SIGLEN_DEFAULT, 1, SIGLEN_MAX, + offsetof(GISTIntArrayBigOptions, siglen)); + + PG_RETURN_VOID(); +} -- cgit v1.2.3