You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

pgstatapprox.c 8.5KB

2 years ago
When updating reltuples after ANALYZE, just extrapolate from our sample. The existing logic for updating pg_class.reltuples trusted the sampling results only for the pages ANALYZE actually visited, preferring to believe the previous tuple density estimate for all the unvisited pages. While there's some rationale for doing that for VACUUM (first that VACUUM is likely to visit a very nonrandom subset of pages, and second that we know for sure that the unvisited pages did not change), there's no such rationale for ANALYZE: by assumption, it's looked at an unbiased random sample of the table's pages. Furthermore, in a very large table ANALYZE will have examined only a tiny fraction of the table's pages, meaning it cannot slew the overall density estimate very far at all. In a table that is physically growing, this causes reltuples to increase nearly proportionally to the change in relpages, regardless of what is actually happening in the table. This has been observed to cause reltuples to become so much larger than reality that it effectively shuts off autovacuum, whose threshold for doing anything is a fraction of reltuples. (Getting to the point where that would happen seems to require some additional, not well understood, conditions. But it's undeniable that if reltuples is seriously off in a large table, ANALYZE alone will not fix it in any reasonable number of iterations, especially not if the table is continuing to grow.) Hence, restrict the use of vac_estimate_reltuples() to VACUUM alone, and in ANALYZE, just extrapolate from the sample pages on the assumption that they provide an accurate model of the whole table. If, by very bad luck, they don't, at least another ANALYZE will fix it; in the old logic a single bad estimate could cause problems indefinitely. In HEAD, let's remove vac_estimate_reltuples' is_analyze argument altogether; it was never used for anything and now it's totally pointless. But keep it in the back branches, in case any third-party code is calling this function. Per bug #15005. Back-patch to all supported branches. David Gould, reviewed by Alexander Kuzmenkov, cosmetic changes by me Discussion: https://postgr.es/m/20180117164916.3fdcf2e9@engels
1 year ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309
  1. /*-------------------------------------------------------------------------
  2. *
  3. * pgstatapprox.c
  4. * Bloat estimation functions
  5. *
  6. * Copyright (c) 2014-2019, PostgreSQL Global Development Group
  7. *
  8. * IDENTIFICATION
  9. * contrib/pgstattuple/pgstatapprox.c
  10. *
  11. *-------------------------------------------------------------------------
  12. */
  13. #include "postgres.h"
  14. #include "access/visibilitymap.h"
  15. #include "access/transam.h"
  16. #include "access/xact.h"
  17. #include "access/multixact.h"
  18. #include "access/htup_details.h"
  19. #include "catalog/namespace.h"
  20. #include "funcapi.h"
  21. #include "miscadmin.h"
  22. #include "storage/bufmgr.h"
  23. #include "storage/freespace.h"
  24. #include "storage/procarray.h"
  25. #include "storage/lmgr.h"
  26. #include "utils/builtins.h"
  27. #include "utils/tqual.h"
  28. #include "commands/vacuum.h"
  29. PG_FUNCTION_INFO_V1(pgstattuple_approx);
  30. PG_FUNCTION_INFO_V1(pgstattuple_approx_v1_5);
  31. Datum pgstattuple_approx_internal(Oid relid, FunctionCallInfo fcinfo);
  32. typedef struct output_type
  33. {
  34. uint64 table_len;
  35. uint64 scanned_percent;
  36. uint64 tuple_count;
  37. uint64 tuple_len;
  38. double tuple_percent;
  39. uint64 dead_tuple_count;
  40. uint64 dead_tuple_len;
  41. double dead_tuple_percent;
  42. uint64 free_space;
  43. double free_percent;
  44. } output_type;
  45. #define NUM_OUTPUT_COLUMNS 10
  46. /*
  47. * This function takes an already open relation and scans its pages,
  48. * skipping those that have the corresponding visibility map bit set.
  49. * For pages we skip, we find the free space from the free space map
  50. * and approximate tuple_len on that basis. For the others, we count
  51. * the exact number of dead tuples etc.
  52. *
  53. * This scan is loosely based on vacuumlazy.c:lazy_scan_heap(), but
  54. * we do not try to avoid skipping single pages.
  55. */
  56. static void
  57. statapprox_heap(Relation rel, output_type *stat)
  58. {
  59. BlockNumber scanned,
  60. nblocks,
  61. blkno;
  62. Buffer vmbuffer = InvalidBuffer;
  63. BufferAccessStrategy bstrategy;
  64. TransactionId OldestXmin;
  65. OldestXmin = GetOldestXmin(rel, PROCARRAY_FLAGS_VACUUM);
  66. bstrategy = GetAccessStrategy(BAS_BULKREAD);
  67. nblocks = RelationGetNumberOfBlocks(rel);
  68. scanned = 0;
  69. for (blkno = 0; blkno < nblocks; blkno++)
  70. {
  71. Buffer buf;
  72. Page page;
  73. OffsetNumber offnum,
  74. maxoff;
  75. Size freespace;
  76. CHECK_FOR_INTERRUPTS();
  77. /*
  78. * If the page has only visible tuples, then we can find out the free
  79. * space from the FSM and move on.
  80. */
  81. if (VM_ALL_VISIBLE(rel, blkno, &vmbuffer))
  82. {
  83. freespace = GetRecordedFreeSpace(rel, blkno);
  84. stat->tuple_len += BLCKSZ - freespace;
  85. stat->free_space += freespace;
  86. continue;
  87. }
  88. buf = ReadBufferExtended(rel, MAIN_FORKNUM, blkno,
  89. RBM_NORMAL, bstrategy);
  90. LockBuffer(buf, BUFFER_LOCK_SHARE);
  91. page = BufferGetPage(buf);
  92. /*
  93. * It's not safe to call PageGetHeapFreeSpace() on new pages, so we
  94. * treat them as being free space for our purposes.
  95. */
  96. if (!PageIsNew(page))
  97. stat->free_space += PageGetHeapFreeSpace(page);
  98. else
  99. stat->free_space += BLCKSZ - SizeOfPageHeaderData;
  100. /* We may count the page as scanned even if it's new/empty */
  101. scanned++;
  102. if (PageIsNew(page) || PageIsEmpty(page))
  103. {
  104. UnlockReleaseBuffer(buf);
  105. continue;
  106. }
  107. /*
  108. * Look at each tuple on the page and decide whether it's live or
  109. * dead, then count it and its size. Unlike lazy_scan_heap, we can
  110. * afford to ignore problems and special cases.
  111. */
  112. maxoff = PageGetMaxOffsetNumber(page);
  113. for (offnum = FirstOffsetNumber;
  114. offnum <= maxoff;
  115. offnum = OffsetNumberNext(offnum))
  116. {
  117. ItemId itemid;
  118. HeapTupleData tuple;
  119. itemid = PageGetItemId(page, offnum);
  120. if (!ItemIdIsUsed(itemid) || ItemIdIsRedirected(itemid) ||
  121. ItemIdIsDead(itemid))
  122. {
  123. continue;
  124. }
  125. Assert(ItemIdIsNormal(itemid));
  126. ItemPointerSet(&(tuple.t_self), blkno, offnum);
  127. tuple.t_data = (HeapTupleHeader) PageGetItem(page, itemid);
  128. tuple.t_len = ItemIdGetLength(itemid);
  129. tuple.t_tableOid = RelationGetRelid(rel);
  130. /*
  131. * We follow VACUUM's lead in counting INSERT_IN_PROGRESS tuples
  132. * as "dead" while DELETE_IN_PROGRESS tuples are "live". We don't
  133. * bother distinguishing tuples inserted/deleted by our own
  134. * transaction.
  135. */
  136. switch (HeapTupleSatisfiesVacuum(&tuple, OldestXmin, buf))
  137. {
  138. case HEAPTUPLE_LIVE:
  139. case HEAPTUPLE_DELETE_IN_PROGRESS:
  140. stat->tuple_len += tuple.t_len;
  141. stat->tuple_count++;
  142. break;
  143. case HEAPTUPLE_DEAD:
  144. case HEAPTUPLE_RECENTLY_DEAD:
  145. case HEAPTUPLE_INSERT_IN_PROGRESS:
  146. stat->dead_tuple_len += tuple.t_len;
  147. stat->dead_tuple_count++;
  148. break;
  149. default:
  150. elog(ERROR, "unexpected HeapTupleSatisfiesVacuum result");
  151. break;
  152. }
  153. }
  154. UnlockReleaseBuffer(buf);
  155. }
  156. stat->table_len = (uint64) nblocks * BLCKSZ;
  157. /*
  158. * We don't know how many tuples are in the pages we didn't scan, so
  159. * extrapolate the live-tuple count to the whole table in the same way
  160. * that VACUUM does. (Like VACUUM, we're not taking a random sample, so
  161. * just extrapolating linearly seems unsafe.) There should be no dead
  162. * tuples in all-visible pages, so no correction is needed for that, and
  163. * we already accounted for the space in those pages, too.
  164. */
  165. stat->tuple_count = vac_estimate_reltuples(rel, nblocks, scanned,
  166. stat->tuple_count);
  167. /*
  168. * Calculate percentages if the relation has one or more pages.
  169. */
  170. if (nblocks != 0)
  171. {
  172. stat->scanned_percent = 100 * scanned / nblocks;
  173. stat->tuple_percent = 100.0 * stat->tuple_len / stat->table_len;
  174. stat->dead_tuple_percent = 100.0 * stat->dead_tuple_len / stat->table_len;
  175. stat->free_percent = 100.0 * stat->free_space / stat->table_len;
  176. }
  177. if (BufferIsValid(vmbuffer))
  178. {
  179. ReleaseBuffer(vmbuffer);
  180. vmbuffer = InvalidBuffer;
  181. }
  182. }
  183. /*
  184. * Returns estimated live/dead tuple statistics for the given relid.
  185. *
  186. * The superuser() check here must be kept as the library might be upgraded
  187. * without the extension being upgraded, meaning that in pre-1.5 installations
  188. * these functions could be called by any user.
  189. */
  190. Datum
  191. pgstattuple_approx(PG_FUNCTION_ARGS)
  192. {
  193. Oid relid = PG_GETARG_OID(0);
  194. if (!superuser())
  195. ereport(ERROR,
  196. (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
  197. (errmsg("must be superuser to use pgstattuple functions"))));
  198. PG_RETURN_DATUM(pgstattuple_approx_internal(relid, fcinfo));
  199. }
  200. /*
  201. * As of pgstattuple version 1.5, we no longer need to check if the user
  202. * is a superuser because we REVOKE EXECUTE on the SQL function from PUBLIC.
  203. * Users can then grant access to it based on their policies.
  204. *
  205. * Otherwise identical to pgstattuple_approx (above).
  206. */
  207. Datum
  208. pgstattuple_approx_v1_5(PG_FUNCTION_ARGS)
  209. {
  210. Oid relid = PG_GETARG_OID(0);
  211. PG_RETURN_DATUM(pgstattuple_approx_internal(relid, fcinfo));
  212. }
  213. Datum
  214. pgstattuple_approx_internal(Oid relid, FunctionCallInfo fcinfo)
  215. {
  216. Relation rel;
  217. output_type stat = {0};
  218. TupleDesc tupdesc;
  219. bool nulls[NUM_OUTPUT_COLUMNS];
  220. Datum values[NUM_OUTPUT_COLUMNS];
  221. HeapTuple ret;
  222. int i = 0;
  223. if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
  224. elog(ERROR, "return type must be a row type");
  225. if (tupdesc->natts != NUM_OUTPUT_COLUMNS)
  226. elog(ERROR, "incorrect number of output arguments");
  227. rel = relation_open(relid, AccessShareLock);
  228. /*
  229. * Reject attempts to read non-local temporary relations; we would be
  230. * likely to get wrong data since we have no visibility into the owning
  231. * session's local buffers.
  232. */
  233. if (RELATION_IS_OTHER_TEMP(rel))
  234. ereport(ERROR,
  235. (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
  236. errmsg("cannot access temporary tables of other sessions")));
  237. /*
  238. * We support only ordinary relations and materialised views, because we
  239. * depend on the visibility map and free space map for our estimates about
  240. * unscanned pages.
  241. */
  242. if (!(rel->rd_rel->relkind == RELKIND_RELATION ||
  243. rel->rd_rel->relkind == RELKIND_MATVIEW))
  244. ereport(ERROR,
  245. (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
  246. errmsg("\"%s\" is not a table or materialized view",
  247. RelationGetRelationName(rel))));
  248. statapprox_heap(rel, &stat);
  249. relation_close(rel, AccessShareLock);
  250. memset(nulls, 0, sizeof(nulls));
  251. values[i++] = Int64GetDatum(stat.table_len);
  252. values[i++] = Float8GetDatum(stat.scanned_percent);
  253. values[i++] = Int64GetDatum(stat.tuple_count);
  254. values[i++] = Int64GetDatum(stat.tuple_len);
  255. values[i++] = Float8GetDatum(stat.tuple_percent);
  256. values[i++] = Int64GetDatum(stat.dead_tuple_count);
  257. values[i++] = Int64GetDatum(stat.dead_tuple_len);
  258. values[i++] = Float8GetDatum(stat.dead_tuple_percent);
  259. values[i++] = Int64GetDatum(stat.free_space);
  260. values[i++] = Float8GetDatum(stat.free_percent);
  261. ret = heap_form_tuple(tupdesc, values, nulls);
  262. return HeapTupleGetDatum(ret);
  263. }