LCOV - code coverage report
Current view: top level - src/backend/tsearch - ts_typanalyze.c (source / functions) Coverage Total Hit
Test: Code coverage Lines: 84.2 % 177 149
Test Date: 2026-01-26 10:56:24 Functions: 87.5 % 8 7
Legend: Lines:     hit not hit
Branches: + taken - not taken # not executed
Branches: 52.8 % 53 28

             Branch data     Line data    Source code
       1                 :             : /*-------------------------------------------------------------------------
       2                 :             :  *
       3                 :             :  * ts_typanalyze.c
       4                 :             :  *        functions for gathering statistics from tsvector columns
       5                 :             :  *
       6                 :             :  * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
       7                 :             :  *
       8                 :             :  *
       9                 :             :  * IDENTIFICATION
      10                 :             :  *        src/backend/tsearch/ts_typanalyze.c
      11                 :             :  *
      12                 :             :  *-------------------------------------------------------------------------
      13                 :             :  */
      14                 :             : #include "postgres.h"
      15                 :             : 
      16                 :             : #include "catalog/pg_collation.h"
      17                 :             : #include "catalog/pg_operator.h"
      18                 :             : #include "commands/vacuum.h"
      19                 :             : #include "common/hashfn.h"
      20                 :             : #include "tsearch/ts_type.h"
      21                 :             : #include "utils/builtins.h"
      22                 :             : #include "varatt.h"
      23                 :             : 
      24                 :             : 
      25                 :             : /* A hash key for lexemes */
      26                 :             : typedef struct
      27                 :             : {
      28                 :             :         char       *lexeme;                     /* lexeme (not NULL terminated!) */
      29                 :             :         int                     length;                 /* its length in bytes */
      30                 :             : } LexemeHashKey;
      31                 :             : 
      32                 :             : /* A hash table entry for the Lossy Counting algorithm */
      33                 :             : typedef struct
      34                 :             : {
      35                 :             :         LexemeHashKey key;                      /* This is 'e' from the LC algorithm. */
      36                 :             :         int                     frequency;              /* This is 'f'. */
      37                 :             :         int                     delta;                  /* And this is 'delta'. */
      38                 :             : } TrackItem;
      39                 :             : 
      40                 :             : static void compute_tsvector_stats(VacAttrStats *stats,
      41                 :             :                                                                    AnalyzeAttrFetchFunc fetchfunc,
      42                 :             :                                                                    int samplerows,
      43                 :             :                                                                    double totalrows);
      44                 :             : static void prune_lexemes_hashtable(HTAB *lexemes_tab, int b_current);
      45                 :             : static uint32 lexeme_hash(const void *key, Size keysize);
      46                 :             : static int      lexeme_match(const void *key1, const void *key2, Size keysize);
      47                 :             : static int      lexeme_compare(const void *key1, const void *key2);
      48                 :             : static int      trackitem_compare_frequencies_desc(const void *e1, const void *e2,
      49                 :             :                                                                                            void *arg);
      50                 :             : static int      trackitem_compare_lexemes(const void *e1, const void *e2,
      51                 :             :                                                                           void *arg);
      52                 :             : 
      53                 :             : 
      54                 :             : /*
      55                 :             :  *      ts_typanalyze -- a custom typanalyze function for tsvector columns
      56                 :             :  */
      57                 :             : Datum
      58                 :           1 : ts_typanalyze(PG_FUNCTION_ARGS)
      59                 :             : {
      60                 :           1 :         VacAttrStats *stats = (VacAttrStats *) PG_GETARG_POINTER(0);
      61                 :             : 
      62                 :             :         /* If the attstattarget column is negative, use the default value */
      63         [ -  + ]:           1 :         if (stats->attstattarget < 0)
      64                 :           1 :                 stats->attstattarget = default_statistics_target;
      65                 :             : 
      66                 :           1 :         stats->compute_stats = compute_tsvector_stats;
      67                 :             :         /* see comment about the choice of minrows in commands/analyze.c */
      68                 :           1 :         stats->minrows = 300 * stats->attstattarget;
      69                 :             : 
      70                 :           2 :         PG_RETURN_BOOL(true);
      71                 :           1 : }
      72                 :             : 
      73                 :             : /*
      74                 :             :  *      compute_tsvector_stats() -- compute statistics for a tsvector column
      75                 :             :  *
      76                 :             :  *      This function computes statistics that are useful for determining @@
      77                 :             :  *      operations' selectivity, along with the fraction of non-null rows and
      78                 :             :  *      average width.
      79                 :             :  *
      80                 :             :  *      Instead of finding the most common values, as we do for most datatypes,
      81                 :             :  *      we're looking for the most common lexemes. This is more useful, because
      82                 :             :  *      there most probably won't be any two rows with the same tsvector and thus
      83                 :             :  *      the notion of a MCV is a bit bogus with this datatype. With a list of the
      84                 :             :  *      most common lexemes we can do a better job at figuring out @@ selectivity.
      85                 :             :  *
      86                 :             :  *      For the same reasons we assume that tsvector columns are unique when
      87                 :             :  *      determining the number of distinct values.
      88                 :             :  *
      89                 :             :  *      The algorithm used is Lossy Counting, as proposed in the paper "Approximate
      90                 :             :  *      frequency counts over data streams" by G. S. Manku and R. Motwani, in
      91                 :             :  *      Proceedings of the 28th International Conference on Very Large Data Bases,
      92                 :             :  *      Hong Kong, China, August 2002, section 4.2. The paper is available at
      93                 :             :  *      http://www.vldb.org/conf/2002/S10P03.pdf
      94                 :             :  *
      95                 :             :  *      The Lossy Counting (aka LC) algorithm goes like this:
      96                 :             :  *      Let s be the threshold frequency for an item (the minimum frequency we
      97                 :             :  *      are interested in) and epsilon the error margin for the frequency. Let D
      98                 :             :  *      be a set of triples (e, f, delta), where e is an element value, f is that
      99                 :             :  *      element's frequency (actually, its current occurrence count) and delta is
     100                 :             :  *      the maximum error in f. We start with D empty and process the elements in
     101                 :             :  *      batches of size w. (The batch size is also known as "bucket size" and is
     102                 :             :  *      equal to 1/epsilon.) Let the current batch number be b_current, starting
     103                 :             :  *      with 1. For each element e we either increment its f count, if it's
     104                 :             :  *      already in D, or insert a new triple into D with values (e, 1, b_current
     105                 :             :  *      - 1). After processing each batch we prune D, by removing from it all
     106                 :             :  *      elements with f + delta <= b_current.  After the algorithm finishes we
     107                 :             :  *      suppress all elements from D that do not satisfy f >= (s - epsilon) * N,
     108                 :             :  *      where N is the total number of elements in the input.  We emit the
     109                 :             :  *      remaining elements with estimated frequency f/N.  The LC paper proves
     110                 :             :  *      that this algorithm finds all elements with true frequency at least s,
     111                 :             :  *      and that no frequency is overestimated or is underestimated by more than
     112                 :             :  *      epsilon.  Furthermore, given reasonable assumptions about the input
     113                 :             :  *      distribution, the required table size is no more than about 7 times w.
     114                 :             :  *
     115                 :             :  *      We set s to be the estimated frequency of the K'th word in a natural
     116                 :             :  *      language's frequency table, where K is the target number of entries in
     117                 :             :  *      the MCELEM array plus an arbitrary constant, meant to reflect the fact
     118                 :             :  *      that the most common words in any language would usually be stopwords
     119                 :             :  *      so we will not actually see them in the input.  We assume that the
     120                 :             :  *      distribution of word frequencies (including the stopwords) follows Zipf's
     121                 :             :  *      law with an exponent of 1.
     122                 :             :  *
     123                 :             :  *      Assuming Zipfian distribution, the frequency of the K'th word is equal
     124                 :             :  *      to 1/(K * H(W)) where H(n) is 1/2 + 1/3 + ... + 1/n and W is the number of
     125                 :             :  *      words in the language.  Putting W as one million, we get roughly 0.07/K.
     126                 :             :  *      Assuming top 10 words are stopwords gives s = 0.07/(K + 10).  We set
     127                 :             :  *      epsilon = s/10, which gives bucket width w = (K + 10)/0.007 and
     128                 :             :  *      maximum expected hashtable size of about 1000 * (K + 10).
     129                 :             :  *
     130                 :             :  *      Note: in the above discussion, s, epsilon, and f/N are in terms of a
     131                 :             :  *      lexeme's frequency as a fraction of all lexemes seen in the input.
     132                 :             :  *      However, what we actually want to store in the finished pg_statistic
     133                 :             :  *      entry is each lexeme's frequency as a fraction of all rows that it occurs
     134                 :             :  *      in.  Assuming that the input tsvectors are correctly constructed, no
     135                 :             :  *      lexeme occurs more than once per tsvector, so the final count f is a
     136                 :             :  *      correct estimate of the number of input tsvectors it occurs in, and we
     137                 :             :  *      need only change the divisor from N to nonnull_cnt to get the number we
     138                 :             :  *      want.
     139                 :             :  */
     140                 :             : static void
     141                 :           1 : compute_tsvector_stats(VacAttrStats *stats,
     142                 :             :                                            AnalyzeAttrFetchFunc fetchfunc,
     143                 :             :                                            int samplerows,
     144                 :             :                                            double totalrows)
     145                 :             : {
     146                 :           1 :         int                     num_mcelem;
     147                 :           1 :         int                     null_cnt = 0;
     148                 :           1 :         double          total_width = 0;
     149                 :             : 
     150                 :             :         /* This is D from the LC algorithm. */
     151                 :           1 :         HTAB       *lexemes_tab;
     152                 :           1 :         HASHCTL         hash_ctl;
     153                 :           1 :         HASH_SEQ_STATUS scan_status;
     154                 :             : 
     155                 :             :         /* This is the current bucket number from the LC algorithm */
     156                 :           1 :         int                     b_current;
     157                 :             : 
     158                 :             :         /* This is 'w' from the LC algorithm */
     159                 :           1 :         int                     bucket_width;
     160                 :           1 :         int                     vector_no,
     161                 :             :                                 lexeme_no;
     162                 :           1 :         LexemeHashKey hash_key;
     163                 :             : 
     164                 :             :         /*
     165                 :             :          * We want statistics_target * 10 lexemes in the MCELEM array.  This
     166                 :             :          * multiplier is pretty arbitrary, but is meant to reflect the fact that
     167                 :             :          * the number of individual lexeme values tracked in pg_statistic ought to
     168                 :             :          * be more than the number of values for a simple scalar column.
     169                 :             :          */
     170                 :           1 :         num_mcelem = stats->attstattarget * 10;
     171                 :             : 
     172                 :             :         /*
     173                 :             :          * We set bucket width equal to (num_mcelem + 10) / 0.007 as per the
     174                 :             :          * comment above.
     175                 :             :          */
     176                 :           1 :         bucket_width = (num_mcelem + 10) * 1000 / 7;
     177                 :             : 
     178                 :             :         /*
     179                 :             :          * Create the hashtable. It will be in local memory, so we don't need to
     180                 :             :          * worry about overflowing the initial size. Also we don't need to pay any
     181                 :             :          * attention to locking and memory management.
     182                 :             :          */
     183                 :           1 :         hash_ctl.keysize = sizeof(LexemeHashKey);
     184                 :           1 :         hash_ctl.entrysize = sizeof(TrackItem);
     185                 :           1 :         hash_ctl.hash = lexeme_hash;
     186                 :           1 :         hash_ctl.match = lexeme_match;
     187                 :           1 :         hash_ctl.hcxt = CurrentMemoryContext;
     188                 :           1 :         lexemes_tab = hash_create("Analyzed lexemes table",
     189                 :           1 :                                                           num_mcelem,
     190                 :             :                                                           &hash_ctl,
     191                 :             :                                                           HASH_ELEM | HASH_FUNCTION | HASH_COMPARE | HASH_CONTEXT);
     192                 :             : 
     193                 :             :         /* Initialize counters. */
     194                 :           1 :         b_current = 1;
     195                 :           1 :         lexeme_no = 0;
     196                 :             : 
     197                 :             :         /* Loop over the tsvectors. */
     198         [ +  + ]:         509 :         for (vector_no = 0; vector_no < samplerows; vector_no++)
     199                 :             :         {
     200                 :         508 :                 Datum           value;
     201                 :         508 :                 bool            isnull;
     202                 :         508 :                 TSVector        vector;
     203                 :         508 :                 WordEntry  *curentryptr;
     204                 :         508 :                 char       *lexemesptr;
     205                 :         508 :                 int                     j;
     206                 :             : 
     207                 :         508 :                 vacuum_delay_point(true);
     208                 :             : 
     209                 :         508 :                 value = fetchfunc(stats, vector_no, &isnull);
     210                 :             : 
     211                 :             :                 /*
     212                 :             :                  * Check for null/nonnull.
     213                 :             :                  */
     214         [ -  + ]:         508 :                 if (isnull)
     215                 :             :                 {
     216                 :           0 :                         null_cnt++;
     217                 :           0 :                         continue;
     218                 :             :                 }
     219                 :             : 
     220                 :             :                 /*
     221                 :             :                  * Add up widths for average-width calculation.  Since it's a
     222                 :             :                  * tsvector, we know it's varlena.  As in the regular
     223                 :             :                  * compute_minimal_stats function, we use the toasted width for this
     224                 :             :                  * calculation.
     225                 :             :                  */
     226                 :         508 :                 total_width += VARSIZE_ANY(DatumGetPointer(value));
     227                 :             : 
     228                 :             :                 /*
     229                 :             :                  * Now detoast the tsvector if needed.
     230                 :             :                  */
     231                 :         508 :                 vector = DatumGetTSVector(value);
     232                 :             : 
     233                 :             :                 /*
     234                 :             :                  * We loop through the lexemes in the tsvector and add them to our
     235                 :             :                  * tracking hashtable.
     236                 :             :                  */
     237                 :         508 :                 lexemesptr = STRPTR(vector);
     238                 :         508 :                 curentryptr = ARRPTR(vector);
     239         [ +  + ]:       29308 :                 for (j = 0; j < vector->size; j++)
     240                 :             :                 {
     241                 :       28800 :                         TrackItem  *item;
     242                 :       28800 :                         bool            found;
     243                 :             : 
     244                 :             :                         /*
     245                 :             :                          * Construct a hash key.  The key points into the (detoasted)
     246                 :             :                          * tsvector value at this point, but if a new entry is created, we
     247                 :             :                          * make a copy of it.  This way we can free the tsvector value
     248                 :             :                          * once we've processed all its lexemes.
     249                 :             :                          */
     250                 :       28800 :                         hash_key.lexeme = lexemesptr + curentryptr->pos;
     251                 :       28800 :                         hash_key.length = curentryptr->len;
     252                 :             : 
     253                 :             :                         /* Lookup current lexeme in hashtable, adding it if new */
     254                 :       28800 :                         item = (TrackItem *) hash_search(lexemes_tab,
     255                 :             :                                                                                          &hash_key,
     256                 :             :                                                                                          HASH_ENTER, &found);
     257                 :             : 
     258         [ +  + ]:       28800 :                         if (found)
     259                 :             :                         {
     260                 :             :                                 /* The lexeme is already on the tracking list */
     261                 :       27659 :                                 item->frequency++;
     262                 :       27659 :                         }
     263                 :             :                         else
     264                 :             :                         {
     265                 :             :                                 /* Initialize new tracking list element */
     266                 :        1141 :                                 item->frequency = 1;
     267                 :        1141 :                                 item->delta = b_current - 1;
     268                 :             : 
     269                 :        1141 :                                 item->key.lexeme = palloc(hash_key.length);
     270                 :        1141 :                                 memcpy(item->key.lexeme, hash_key.lexeme, hash_key.length);
     271                 :             :                         }
     272                 :             : 
     273                 :             :                         /* lexeme_no is the number of elements processed (ie N) */
     274                 :       28800 :                         lexeme_no++;
     275                 :             : 
     276                 :             :                         /* We prune the D structure after processing each bucket */
     277         [ +  - ]:       28800 :                         if (lexeme_no % bucket_width == 0)
     278                 :             :                         {
     279                 :           0 :                                 prune_lexemes_hashtable(lexemes_tab, b_current);
     280                 :           0 :                                 b_current++;
     281                 :           0 :                         }
     282                 :             : 
     283                 :             :                         /* Advance to the next WordEntry in the tsvector */
     284                 :       28800 :                         curentryptr++;
     285                 :       28800 :                 }
     286                 :             : 
     287                 :             :                 /* If the vector was toasted, free the detoasted copy. */
     288         [ +  + ]:         508 :                 if (TSVectorGetDatum(vector) != value)
     289                 :          64 :                         pfree(vector);
     290      [ +  -  - ]:         508 :         }
     291                 :             : 
     292                 :             :         /* We can only compute real stats if we found some non-null values. */
     293         [ +  - ]:           1 :         if (null_cnt < samplerows)
     294                 :             :         {
     295                 :           1 :                 int                     nonnull_cnt = samplerows - null_cnt;
     296                 :           1 :                 int                     i;
     297                 :           1 :                 TrackItem **sort_table;
     298                 :           1 :                 TrackItem  *item;
     299                 :           1 :                 int                     track_len;
     300                 :           1 :                 int                     cutoff_freq;
     301                 :           1 :                 int                     minfreq,
     302                 :             :                                         maxfreq;
     303                 :             : 
     304                 :           1 :                 stats->stats_valid = true;
     305                 :             :                 /* Do the simple null-frac and average width stats */
     306                 :           1 :                 stats->stanullfrac = (double) null_cnt / (double) samplerows;
     307                 :           1 :                 stats->stawidth = total_width / (double) nonnull_cnt;
     308                 :             : 
     309                 :             :                 /* Assume it's a unique column (see notes above) */
     310                 :           1 :                 stats->stadistinct = -1.0 * (1.0 - stats->stanullfrac);
     311                 :             : 
     312                 :             :                 /*
     313                 :             :                  * Construct an array of the interesting hashtable items, that is,
     314                 :             :                  * those meeting the cutoff frequency (s - epsilon)*N.  Also identify
     315                 :             :                  * the maximum frequency among these items.
     316                 :             :                  *
     317                 :             :                  * Since epsilon = s/10 and bucket_width = 1/epsilon, the cutoff
     318                 :             :                  * frequency is 9*N / bucket_width.
     319                 :             :                  */
     320                 :           1 :                 cutoff_freq = 9 * lexeme_no / bucket_width;
     321                 :             : 
     322                 :           1 :                 i = hash_get_num_entries(lexemes_tab);  /* surely enough space */
     323                 :           1 :                 sort_table = palloc_array(TrackItem *, i);
     324                 :             : 
     325                 :           1 :                 hash_seq_init(&scan_status, lexemes_tab);
     326                 :           1 :                 track_len = 0;
     327                 :           1 :                 maxfreq = 0;
     328         [ +  + ]:        1142 :                 while ((item = (TrackItem *) hash_seq_search(&scan_status)) != NULL)
     329                 :             :                 {
     330         [ +  + ]:        1141 :                         if (item->frequency > cutoff_freq)
     331                 :             :                         {
     332                 :        1053 :                                 sort_table[track_len++] = item;
     333         [ +  + ]:        1053 :                                 maxfreq = Max(maxfreq, item->frequency);
     334                 :        1053 :                         }
     335                 :             :                 }
     336         [ +  - ]:           1 :                 Assert(track_len <= i);
     337                 :             : 
     338                 :             :                 /* emit some statistics for debug purposes */
     339   [ -  +  -  + ]:           1 :                 elog(DEBUG3, "tsvector_stats: target # mces = %d, bucket width = %d, "
     340                 :             :                          "# lexemes = %d, hashtable size = %d, usable entries = %d",
     341                 :             :                          num_mcelem, bucket_width, lexeme_no, i, track_len);
     342                 :             : 
     343                 :             :                 /*
     344                 :             :                  * If we obtained more lexemes than we really want, get rid of those
     345                 :             :                  * with least frequencies.  The easiest way is to qsort the array into
     346                 :             :                  * descending frequency order and truncate the array.
     347                 :             :                  *
     348                 :             :                  * If we did not find more elements than we want, then it is safe to
     349                 :             :                  * assume that the stored MCE array will contain every element with
     350                 :             :                  * frequency above the cutoff.  In that case, rather than storing the
     351                 :             :                  * smallest frequency we are keeping, we want to store the minimum
     352                 :             :                  * frequency that would have been accepted as a valid MCE.  The
     353                 :             :                  * selectivity functions can assume that that is an upper bound on the
     354                 :             :                  * frequency of elements not present in the array.
     355                 :             :                  *
     356                 :             :                  * If we found no candidate MCEs at all, we still want to record the
     357                 :             :                  * cutoff frequency, since it's still valid to assume that no element
     358                 :             :                  * has frequency more than that.
     359                 :             :                  */
     360         [ +  - ]:           1 :                 if (num_mcelem < track_len)
     361                 :             :                 {
     362                 :           1 :                         qsort_interruptible(sort_table, track_len, sizeof(TrackItem *),
     363                 :             :                                                                 trackitem_compare_frequencies_desc, NULL);
     364                 :             :                         /* set minfreq to the smallest frequency we're keeping */
     365                 :           1 :                         minfreq = sort_table[num_mcelem - 1]->frequency;
     366                 :           1 :                 }
     367                 :             :                 else
     368                 :             :                 {
     369                 :           0 :                         num_mcelem = track_len;
     370                 :             :                         /* set minfreq to the minimum frequency above the cutoff */
     371                 :           0 :                         minfreq = cutoff_freq + 1;
     372                 :             :                         /* ensure maxfreq is nonzero, too */
     373         [ #  # ]:           0 :                         if (track_len == 0)
     374                 :           0 :                                 maxfreq = minfreq;
     375                 :             :                 }
     376                 :             : 
     377                 :             :                 /* Generate MCELEM slot entry */
     378         [ -  + ]:           1 :                 if (num_mcelem >= 0)
     379                 :             :                 {
     380                 :           1 :                         MemoryContext old_context;
     381                 :           1 :                         Datum      *mcelem_values;
     382                 :           1 :                         float4     *mcelem_freqs;
     383                 :             : 
     384                 :             :                         /*
     385                 :             :                          * We want to store statistics sorted on the lexeme value using
     386                 :             :                          * first length, then byte-for-byte comparison. The reason for
     387                 :             :                          * doing length comparison first is that we don't care about the
     388                 :             :                          * ordering so long as it's consistent, and comparing lengths
     389                 :             :                          * first gives us a chance to avoid a strncmp() call.
     390                 :             :                          *
     391                 :             :                          * This is different from what we do with scalar statistics --
     392                 :             :                          * they get sorted on frequencies. The rationale is that we
     393                 :             :                          * usually search through most common elements looking for a
     394                 :             :                          * specific value, so we can grab its frequency.  When values are
     395                 :             :                          * presorted we can employ binary search for that.  See
     396                 :             :                          * ts_selfuncs.c for a real usage scenario.
     397                 :             :                          */
     398                 :           1 :                         qsort_interruptible(sort_table, num_mcelem, sizeof(TrackItem *),
     399                 :             :                                                                 trackitem_compare_lexemes, NULL);
     400                 :             : 
     401                 :             :                         /* Must copy the target values into anl_context */
     402                 :           1 :                         old_context = MemoryContextSwitchTo(stats->anl_context);
     403                 :             : 
     404                 :             :                         /*
     405                 :             :                          * We sorted statistics on the lexeme value, but we want to be
     406                 :             :                          * able to find out the minimal and maximal frequency without
     407                 :             :                          * going through all the values.  We keep those two extra
     408                 :             :                          * frequencies in two extra cells in mcelem_freqs.
     409                 :             :                          *
     410                 :             :                          * (Note: the MCELEM statistics slot definition allows for a third
     411                 :             :                          * extra number containing the frequency of nulls, but we don't
     412                 :             :                          * create that for a tsvector column, since null elements aren't
     413                 :             :                          * possible.)
     414                 :             :                          */
     415                 :           1 :                         mcelem_values = palloc_array(Datum, num_mcelem);
     416                 :           1 :                         mcelem_freqs = palloc_array(float4, num_mcelem + 2);
     417                 :             : 
     418                 :             :                         /*
     419                 :             :                          * See comments above about use of nonnull_cnt as the divisor for
     420                 :             :                          * the final frequency estimates.
     421                 :             :                          */
     422         [ +  + ]:        1001 :                         for (i = 0; i < num_mcelem; i++)
     423                 :             :                         {
     424                 :        1000 :                                 TrackItem  *titem = sort_table[i];
     425                 :             : 
     426                 :        1000 :                                 mcelem_values[i] =
     427                 :        2000 :                                         PointerGetDatum(cstring_to_text_with_len(titem->key.lexeme,
     428                 :        1000 :                                                                                                                          titem->key.length));
     429                 :        1000 :                                 mcelem_freqs[i] = (double) titem->frequency / (double) nonnull_cnt;
     430                 :        1000 :                         }
     431                 :           1 :                         mcelem_freqs[i++] = (double) minfreq / (double) nonnull_cnt;
     432                 :           1 :                         mcelem_freqs[i] = (double) maxfreq / (double) nonnull_cnt;
     433                 :           1 :                         MemoryContextSwitchTo(old_context);
     434                 :             : 
     435                 :           1 :                         stats->stakind[0] = STATISTIC_KIND_MCELEM;
     436                 :           1 :                         stats->staop[0] = TextEqualOperator;
     437                 :           1 :                         stats->stacoll[0] = DEFAULT_COLLATION_OID;
     438                 :           1 :                         stats->stanumbers[0] = mcelem_freqs;
     439                 :             :                         /* See above comment about two extra frequency fields */
     440                 :           1 :                         stats->numnumbers[0] = num_mcelem + 2;
     441                 :           1 :                         stats->stavalues[0] = mcelem_values;
     442                 :           1 :                         stats->numvalues[0] = num_mcelem;
     443                 :             :                         /* We are storing text values */
     444                 :           1 :                         stats->statypid[0] = TEXTOID;
     445                 :           1 :                         stats->statyplen[0] = -1;    /* typlen, -1 for varlena */
     446                 :           1 :                         stats->statypbyval[0] = false;
     447                 :           1 :                         stats->statypalign[0] = 'i';
     448                 :           1 :                 }
     449                 :           1 :         }
     450                 :             :         else
     451                 :             :         {
     452                 :             :                 /* We found only nulls; assume the column is entirely null */
     453                 :           0 :                 stats->stats_valid = true;
     454                 :           0 :                 stats->stanullfrac = 1.0;
     455                 :           0 :                 stats->stawidth = 0; /* "unknown" */
     456                 :           0 :                 stats->stadistinct = 0.0;    /* "unknown" */
     457                 :             :         }
     458                 :             : 
     459                 :             :         /*
     460                 :             :          * We don't need to bother cleaning up any of our temporary palloc's. The
     461                 :             :          * hashtable should also go away, as it used a child memory context.
     462                 :             :          */
     463                 :           1 : }
     464                 :             : 
     465                 :             : /*
     466                 :             :  *      A function to prune the D structure from the Lossy Counting algorithm.
     467                 :             :  *      Consult compute_tsvector_stats() for wider explanation.
     468                 :             :  */
     469                 :             : static void
     470                 :           0 : prune_lexemes_hashtable(HTAB *lexemes_tab, int b_current)
     471                 :             : {
     472                 :           0 :         HASH_SEQ_STATUS scan_status;
     473                 :           0 :         TrackItem  *item;
     474                 :             : 
     475                 :           0 :         hash_seq_init(&scan_status, lexemes_tab);
     476         [ #  # ]:           0 :         while ((item = (TrackItem *) hash_seq_search(&scan_status)) != NULL)
     477                 :             :         {
     478         [ #  # ]:           0 :                 if (item->frequency + item->delta <= b_current)
     479                 :             :                 {
     480                 :           0 :                         char       *lexeme = item->key.lexeme;
     481                 :             : 
     482                 :           0 :                         if (hash_search(lexemes_tab, &item->key,
     483         [ #  # ]:           0 :                                                         HASH_REMOVE, NULL) == NULL)
     484   [ #  #  #  # ]:           0 :                                 elog(ERROR, "hash table corrupted");
     485                 :           0 :                         pfree(lexeme);
     486                 :           0 :                 }
     487                 :             :         }
     488                 :           0 : }
     489                 :             : 
     490                 :             : /*
     491                 :             :  * Hash functions for lexemes. They are strings, but not NULL terminated,
     492                 :             :  * so we need a special hash function.
     493                 :             :  */
     494                 :             : static uint32
     495                 :       28800 : lexeme_hash(const void *key, Size keysize)
     496                 :             : {
     497                 :       28800 :         const LexemeHashKey *l = (const LexemeHashKey *) key;
     498                 :             : 
     499                 :       86400 :         return DatumGetUInt32(hash_any((const unsigned char *) l->lexeme,
     500                 :       28800 :                                                                    l->length));
     501                 :       28800 : }
     502                 :             : 
     503                 :             : /*
     504                 :             :  *      Matching function for lexemes, to be used in hashtable lookups.
     505                 :             :  */
     506                 :             : static int
     507                 :       27659 : lexeme_match(const void *key1, const void *key2, Size keysize)
     508                 :             : {
     509                 :             :         /* The keysize parameter is superfluous, the keys store their lengths */
     510                 :       27659 :         return lexeme_compare(key1, key2);
     511                 :             : }
     512                 :             : 
     513                 :             : /*
     514                 :             :  *      Comparison function for lexemes.
     515                 :             :  */
     516                 :             : static int
     517                 :       37838 : lexeme_compare(const void *key1, const void *key2)
     518                 :             : {
     519                 :       37838 :         const LexemeHashKey *d1 = (const LexemeHashKey *) key1;
     520                 :       37838 :         const LexemeHashKey *d2 = (const LexemeHashKey *) key2;
     521                 :             : 
     522                 :             :         /* First, compare by length */
     523         [ -  + ]:       37838 :         if (d1->length > d2->length)
     524                 :           0 :                 return 1;
     525         [ -  + ]:       37838 :         else if (d1->length < d2->length)
     526                 :           0 :                 return -1;
     527                 :             :         /* Lengths are equal, do a byte-by-byte comparison */
     528                 :       37838 :         return strncmp(d1->lexeme, d2->lexeme, d1->length);
     529                 :       37838 : }
     530                 :             : 
     531                 :             : /*
     532                 :             :  *      Comparator for sorting TrackItems on frequencies (descending sort)
     533                 :             :  */
     534                 :             : static int
     535                 :        6410 : trackitem_compare_frequencies_desc(const void *e1, const void *e2, void *arg)
     536                 :             : {
     537                 :        6410 :         const TrackItem *const *t1 = (const TrackItem *const *) e1;
     538                 :        6410 :         const TrackItem *const *t2 = (const TrackItem *const *) e2;
     539                 :             : 
     540                 :       12820 :         return (*t2)->frequency - (*t1)->frequency;
     541                 :        6410 : }
     542                 :             : 
     543                 :             : /*
     544                 :             :  *      Comparator for sorting TrackItems on lexemes
     545                 :             :  */
     546                 :             : static int
     547                 :       10179 : trackitem_compare_lexemes(const void *e1, const void *e2, void *arg)
     548                 :             : {
     549                 :       10179 :         const TrackItem *const *t1 = (const TrackItem *const *) e1;
     550                 :       10179 :         const TrackItem *const *t2 = (const TrackItem *const *) e2;
     551                 :             : 
     552                 :       20358 :         return lexeme_compare(&(*t1)->key, &(*t2)->key);
     553                 :       10179 : }
        

Generated by: LCOV version 2.3.2-1