You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

pg_stat_statements.c 89KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166
  1. /*-------------------------------------------------------------------------
  2. *
  3. * pg_stat_statements.c
  4. * Track statement execution times across a whole database cluster.
  5. *
  6. * Execution costs are totalled for each distinct source query, and kept in
  7. * a shared hashtable. (We track only as many distinct queries as will fit
  8. * in the designated amount of shared memory.)
  9. *
  10. * As of Postgres 9.2, this module normalizes query entries. Normalization
  11. * is a process whereby similar queries, typically differing only in their
  12. * constants (though the exact rules are somewhat more subtle than that) are
  13. * recognized as equivalent, and are tracked as a single entry. This is
  14. * particularly useful for non-prepared queries.
  15. *
  16. * Normalization is implemented by fingerprinting queries, selectively
  17. * serializing those fields of each query tree's nodes that are judged to be
  18. * essential to the query. This is referred to as a query jumble. This is
  19. * distinct from a regular serialization in that various extraneous
  20. * information is ignored as irrelevant or not essential to the query, such
  21. * as the collations of Vars and, most notably, the values of constants.
  22. *
  23. * This jumble is acquired at the end of parse analysis of each query, and
  24. * a 64-bit hash of it is stored into the query's Query.queryId field.
  25. * The server then copies this value around, making it available in plan
  26. * tree(s) generated from the query. The executor can then use this value
  27. * to blame query costs on the proper queryId.
  28. *
  29. * To facilitate presenting entries to users, we create "representative" query
  30. * strings in which constants are replaced with parameter symbols ($n), to
  31. * make it clearer what a normalized entry can represent. To save on shared
  32. * memory, and to avoid having to truncate oversized query strings, we store
  33. * these strings in a temporary external query-texts file. Offsets into this
  34. * file are kept in shared memory.
  35. *
  36. * Note about locking issues: to create or delete an entry in the shared
  37. * hashtable, one must hold pgss->lock exclusively. Modifying any field
  38. * in an entry except the counters requires the same. To look up an entry,
  39. * one must hold the lock shared. To read or update the counters within
  40. * an entry, one must hold the lock shared or exclusive (so the entry doesn't
  41. * disappear!) and also take the entry's mutex spinlock.
  42. * The shared state variable pgss->extent (the next free spot in the external
  43. * query-text file) should be accessed only while holding either the
  44. * pgss->mutex spinlock, or exclusive lock on pgss->lock. We use the mutex to
  45. * allow reserving file space while holding only shared lock on pgss->lock.
  46. * Rewriting the entire external query-text file, eg for garbage collection,
  47. * requires holding pgss->lock exclusively; this allows individual entries
  48. * in the file to be read or written while holding only shared lock.
  49. *
  50. *
  51. * Copyright (c) 2008-2019, PostgreSQL Global Development Group
  52. *
  53. * IDENTIFICATION
  54. * contrib/pg_stat_statements/pg_stat_statements.c
  55. *
  56. *-------------------------------------------------------------------------
  57. */
  58. #include "postgres.h"
  59. #include <math.h>
  60. #include <sys/stat.h>
  61. #include <unistd.h>
  62. #include "access/hash.h"
  63. #include "catalog/pg_authid.h"
  64. #include "executor/instrument.h"
  65. #include "funcapi.h"
  66. #include "mb/pg_wchar.h"
  67. #include "miscadmin.h"
  68. #include "parser/analyze.h"
  69. #include "parser/parsetree.h"
  70. #include "parser/scanner.h"
  71. #include "parser/scansup.h"
  72. #include "pgstat.h"
  73. #include "storage/fd.h"
  74. #include "storage/ipc.h"
  75. #include "storage/spin.h"
  76. #include "tcop/utility.h"
  77. #include "utils/acl.h"
  78. #include "utils/builtins.h"
  79. #include "utils/memutils.h"
  80. PG_MODULE_MAGIC;
  81. /* Location of permanent stats file (valid when database is shut down) */
  82. #define PGSS_DUMP_FILE PGSTAT_STAT_PERMANENT_DIRECTORY "/pg_stat_statements.stat"
  83. /*
  84. * Location of external query text file. We don't keep it in the core
  85. * system's stats_temp_directory. The core system can safely use that GUC
  86. * setting, because the statistics collector temp file paths are set only once
  87. * as part of changing the GUC, but pg_stat_statements has no way of avoiding
  88. * race conditions. Besides, we only expect modest, infrequent I/O for query
  89. * strings, so placing the file on a faster filesystem is not compelling.
  90. */
  91. #define PGSS_TEXT_FILE PG_STAT_TMP_DIR "/pgss_query_texts.stat"
  92. /* Magic number identifying the stats file format */
  93. static const uint32 PGSS_FILE_HEADER = 0x20171004;
  94. /* PostgreSQL major version number, changes in which invalidate all entries */
  95. static const uint32 PGSS_PG_MAJOR_VERSION = PG_VERSION_NUM / 100;
  96. /* XXX: Should USAGE_EXEC reflect execution time and/or buffer usage? */
  97. #define USAGE_EXEC(duration) (1.0)
  98. #define USAGE_INIT (1.0) /* including initial planning */
  99. #define ASSUMED_MEDIAN_INIT (10.0) /* initial assumed median usage */
  100. #define ASSUMED_LENGTH_INIT 1024 /* initial assumed mean query length */
  101. #define USAGE_DECREASE_FACTOR (0.99) /* decreased every entry_dealloc */
  102. #define STICKY_DECREASE_FACTOR (0.50) /* factor for sticky entries */
  103. #define USAGE_DEALLOC_PERCENT 5 /* free this % of entries at once */
  104. #define JUMBLE_SIZE 1024 /* query serialization buffer size */
  105. /*
  106. * Extension version number, for supporting older extension versions' objects
  107. */
  108. typedef enum pgssVersion
  109. {
  110. PGSS_V1_0 = 0,
  111. PGSS_V1_1,
  112. PGSS_V1_2,
  113. PGSS_V1_3
  114. } pgssVersion;
  115. /*
  116. * Hashtable key that defines the identity of a hashtable entry. We separate
  117. * queries by user and by database even if they are otherwise identical.
  118. *
  119. * Right now, this structure contains no padding. If you add any, make sure
  120. * to teach pgss_store() to zero the padding bytes. Otherwise, things will
  121. * break, because pgss_hash is created using HASH_BLOBS, and thus tag_hash
  122. * is used to hash this.
  123. */
  124. typedef struct pgssHashKey
  125. {
  126. Oid userid; /* user OID */
  127. Oid dbid; /* database OID */
  128. uint64 queryid; /* query identifier */
  129. } pgssHashKey;
  130. /*
  131. * The actual stats counters kept within pgssEntry.
  132. */
  133. typedef struct Counters
  134. {
  135. int64 calls; /* # of times executed */
  136. double total_time; /* total execution time, in msec */
  137. double min_time; /* minimum execution time in msec */
  138. double max_time; /* maximum execution time in msec */
  139. double mean_time; /* mean execution time in msec */
  140. double sum_var_time; /* sum of variances in execution time in msec */
  141. int64 rows; /* total # of retrieved or affected rows */
  142. int64 shared_blks_hit; /* # of shared buffer hits */
  143. int64 shared_blks_read; /* # of shared disk blocks read */
  144. int64 shared_blks_dirtied; /* # of shared disk blocks dirtied */
  145. int64 shared_blks_written; /* # of shared disk blocks written */
  146. int64 local_blks_hit; /* # of local buffer hits */
  147. int64 local_blks_read; /* # of local disk blocks read */
  148. int64 local_blks_dirtied; /* # of local disk blocks dirtied */
  149. int64 local_blks_written; /* # of local disk blocks written */
  150. int64 temp_blks_read; /* # of temp blocks read */
  151. int64 temp_blks_written; /* # of temp blocks written */
  152. double blk_read_time; /* time spent reading, in msec */
  153. double blk_write_time; /* time spent writing, in msec */
  154. double usage; /* usage factor */
  155. } Counters;
  156. /*
  157. * Statistics per statement
  158. *
  159. * Note: in event of a failure in garbage collection of the query text file,
  160. * we reset query_offset to zero and query_len to -1. This will be seen as
  161. * an invalid state by qtext_fetch().
  162. */
  163. typedef struct pgssEntry
  164. {
  165. pgssHashKey key; /* hash key of entry - MUST BE FIRST */
  166. Counters counters; /* the statistics for this query */
  167. Size query_offset; /* query text offset in external file */
  168. int query_len; /* # of valid bytes in query string, or -1 */
  169. int encoding; /* query text encoding */
  170. slock_t mutex; /* protects the counters only */
  171. } pgssEntry;
  172. /*
  173. * Global shared state
  174. */
  175. typedef struct pgssSharedState
  176. {
  177. LWLock *lock; /* protects hashtable search/modification */
  178. double cur_median_usage; /* current median usage in hashtable */
  179. Size mean_query_len; /* current mean entry text length */
  180. slock_t mutex; /* protects following fields only: */
  181. Size extent; /* current extent of query file */
  182. int n_writers; /* number of active writers to query file */
  183. int gc_count; /* query file garbage collection cycle count */
  184. } pgssSharedState;
  185. /*
  186. * Struct for tracking locations/lengths of constants during normalization
  187. */
  188. typedef struct pgssLocationLen
  189. {
  190. int location; /* start offset in query text */
  191. int length; /* length in bytes, or -1 to ignore */
  192. } pgssLocationLen;
  193. /*
  194. * Working state for computing a query jumble and producing a normalized
  195. * query string
  196. */
  197. typedef struct pgssJumbleState
  198. {
  199. /* Jumble of current query tree */
  200. unsigned char *jumble;
  201. /* Number of bytes used in jumble[] */
  202. Size jumble_len;
  203. /* Array of locations of constants that should be removed */
  204. pgssLocationLen *clocations;
  205. /* Allocated length of clocations array */
  206. int clocations_buf_size;
  207. /* Current number of valid entries in clocations array */
  208. int clocations_count;
  209. /* highest Param id we've seen, in order to start normalization correctly */
  210. int highest_extern_param_id;
  211. } pgssJumbleState;
  212. /*---- Local variables ----*/
  213. /* Current nesting depth of ExecutorRun+ProcessUtility calls */
  214. static int nested_level = 0;
  215. /* Saved hook values in case of unload */
  216. static shmem_startup_hook_type prev_shmem_startup_hook = NULL;
  217. static post_parse_analyze_hook_type prev_post_parse_analyze_hook = NULL;
  218. static ExecutorStart_hook_type prev_ExecutorStart = NULL;
  219. static ExecutorRun_hook_type prev_ExecutorRun = NULL;
  220. static ExecutorFinish_hook_type prev_ExecutorFinish = NULL;
  221. static ExecutorEnd_hook_type prev_ExecutorEnd = NULL;
  222. static ProcessUtility_hook_type prev_ProcessUtility = NULL;
  223. /* Links to shared memory state */
  224. static pgssSharedState *pgss = NULL;
  225. static HTAB *pgss_hash = NULL;
  226. /*---- GUC variables ----*/
  227. typedef enum
  228. {
  229. PGSS_TRACK_NONE, /* track no statements */
  230. PGSS_TRACK_TOP, /* only top level statements */
  231. PGSS_TRACK_ALL /* all statements, including nested ones */
  232. } PGSSTrackLevel;
  233. static const struct config_enum_entry track_options[] =
  234. {
  235. {"none", PGSS_TRACK_NONE, false},
  236. {"top", PGSS_TRACK_TOP, false},
  237. {"all", PGSS_TRACK_ALL, false},
  238. {NULL, 0, false}
  239. };
  240. static int pgss_max; /* max # statements to track */
  241. static int pgss_track; /* tracking level */
  242. static bool pgss_track_utility; /* whether to track utility commands */
  243. static bool pgss_save; /* whether to save stats across shutdown */
  244. #define pgss_enabled() \
  245. (pgss_track == PGSS_TRACK_ALL || \
  246. (pgss_track == PGSS_TRACK_TOP && nested_level == 0))
  247. #define record_gc_qtexts() \
  248. do { \
  249. volatile pgssSharedState *s = (volatile pgssSharedState *) pgss; \
  250. SpinLockAcquire(&s->mutex); \
  251. s->gc_count++; \
  252. SpinLockRelease(&s->mutex); \
  253. } while(0)
  254. /*---- Function declarations ----*/
  255. void _PG_init(void);
  256. void _PG_fini(void);
  257. PG_FUNCTION_INFO_V1(pg_stat_statements_reset);
  258. PG_FUNCTION_INFO_V1(pg_stat_statements_1_2);
  259. PG_FUNCTION_INFO_V1(pg_stat_statements_1_3);
  260. PG_FUNCTION_INFO_V1(pg_stat_statements);
  261. static void pgss_shmem_startup(void);
  262. static void pgss_shmem_shutdown(int code, Datum arg);
  263. static void pgss_post_parse_analyze(ParseState *pstate, Query *query);
  264. static void pgss_ExecutorStart(QueryDesc *queryDesc, int eflags);
  265. static void pgss_ExecutorRun(QueryDesc *queryDesc,
  266. ScanDirection direction,
  267. uint64 count, bool execute_once);
  268. static void pgss_ExecutorFinish(QueryDesc *queryDesc);
  269. static void pgss_ExecutorEnd(QueryDesc *queryDesc);
  270. static void pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString,
  271. ProcessUtilityContext context, ParamListInfo params,
  272. QueryEnvironment *queryEnv,
  273. DestReceiver *dest, char *completionTag);
  274. static uint64 pgss_hash_string(const char *str, int len);
  275. static void pgss_store(const char *query, uint64 queryId,
  276. int query_location, int query_len,
  277. double total_time, uint64 rows,
  278. const BufferUsage *bufusage,
  279. pgssJumbleState *jstate);
  280. static void pg_stat_statements_internal(FunctionCallInfo fcinfo,
  281. pgssVersion api_version,
  282. bool showtext);
  283. static Size pgss_memsize(void);
  284. static pgssEntry *entry_alloc(pgssHashKey *key, Size query_offset, int query_len,
  285. int encoding, bool sticky);
  286. static void entry_dealloc(void);
  287. static bool qtext_store(const char *query, int query_len,
  288. Size *query_offset, int *gc_count);
  289. static char *qtext_load_file(Size *buffer_size);
  290. static char *qtext_fetch(Size query_offset, int query_len,
  291. char *buffer, Size buffer_size);
  292. static bool need_gc_qtexts(void);
  293. static void gc_qtexts(void);
  294. static void entry_reset(void);
  295. static void AppendJumble(pgssJumbleState *jstate,
  296. const unsigned char *item, Size size);
  297. static void JumbleQuery(pgssJumbleState *jstate, Query *query);
  298. static void JumbleRangeTable(pgssJumbleState *jstate, List *rtable);
  299. static void JumbleExpr(pgssJumbleState *jstate, Node *node);
  300. static void RecordConstLocation(pgssJumbleState *jstate, int location);
  301. static char *generate_normalized_query(pgssJumbleState *jstate, const char *query,
  302. int query_loc, int *query_len_p, int encoding);
  303. static void fill_in_constant_lengths(pgssJumbleState *jstate, const char *query,
  304. int query_loc);
  305. static int comp_location(const void *a, const void *b);
  306. /*
  307. * Module load callback
  308. */
  309. void
  310. _PG_init(void)
  311. {
  312. /*
  313. * In order to create our shared memory area, we have to be loaded via
  314. * shared_preload_libraries. If not, fall out without hooking into any of
  315. * the main system. (We don't throw error here because it seems useful to
  316. * allow the pg_stat_statements functions to be created even when the
  317. * module isn't active. The functions must protect themselves against
  318. * being called then, however.)
  319. */
  320. if (!process_shared_preload_libraries_in_progress)
  321. return;
  322. /*
  323. * Define (or redefine) custom GUC variables.
  324. */
  325. DefineCustomIntVariable("pg_stat_statements.max",
  326. "Sets the maximum number of statements tracked by pg_stat_statements.",
  327. NULL,
  328. &pgss_max,
  329. 5000,
  330. 100,
  331. INT_MAX,
  332. PGC_POSTMASTER,
  333. 0,
  334. NULL,
  335. NULL,
  336. NULL);
  337. DefineCustomEnumVariable("pg_stat_statements.track",
  338. "Selects which statements are tracked by pg_stat_statements.",
  339. NULL,
  340. &pgss_track,
  341. PGSS_TRACK_TOP,
  342. track_options,
  343. PGC_SUSET,
  344. 0,
  345. NULL,
  346. NULL,
  347. NULL);
  348. DefineCustomBoolVariable("pg_stat_statements.track_utility",
  349. "Selects whether utility commands are tracked by pg_stat_statements.",
  350. NULL,
  351. &pgss_track_utility,
  352. true,
  353. PGC_SUSET,
  354. 0,
  355. NULL,
  356. NULL,
  357. NULL);
  358. DefineCustomBoolVariable("pg_stat_statements.save",
  359. "Save pg_stat_statements statistics across server shutdowns.",
  360. NULL,
  361. &pgss_save,
  362. true,
  363. PGC_SIGHUP,
  364. 0,
  365. NULL,
  366. NULL,
  367. NULL);
  368. EmitWarningsOnPlaceholders("pg_stat_statements");
  369. /*
  370. * Request additional shared resources. (These are no-ops if we're not in
  371. * the postmaster process.) We'll allocate or attach to the shared
  372. * resources in pgss_shmem_startup().
  373. */
  374. RequestAddinShmemSpace(pgss_memsize());
  375. RequestNamedLWLockTranche("pg_stat_statements", 1);
  376. /*
  377. * Install hooks.
  378. */
  379. prev_shmem_startup_hook = shmem_startup_hook;
  380. shmem_startup_hook = pgss_shmem_startup;
  381. prev_post_parse_analyze_hook = post_parse_analyze_hook;
  382. post_parse_analyze_hook = pgss_post_parse_analyze;
  383. prev_ExecutorStart = ExecutorStart_hook;
  384. ExecutorStart_hook = pgss_ExecutorStart;
  385. prev_ExecutorRun = ExecutorRun_hook;
  386. ExecutorRun_hook = pgss_ExecutorRun;
  387. prev_ExecutorFinish = ExecutorFinish_hook;
  388. ExecutorFinish_hook = pgss_ExecutorFinish;
  389. prev_ExecutorEnd = ExecutorEnd_hook;
  390. ExecutorEnd_hook = pgss_ExecutorEnd;
  391. prev_ProcessUtility = ProcessUtility_hook;
  392. ProcessUtility_hook = pgss_ProcessUtility;
  393. }
  394. /*
  395. * Module unload callback
  396. */
  397. void
  398. _PG_fini(void)
  399. {
  400. /* Uninstall hooks. */
  401. shmem_startup_hook = prev_shmem_startup_hook;
  402. post_parse_analyze_hook = prev_post_parse_analyze_hook;
  403. ExecutorStart_hook = prev_ExecutorStart;
  404. ExecutorRun_hook = prev_ExecutorRun;
  405. ExecutorFinish_hook = prev_ExecutorFinish;
  406. ExecutorEnd_hook = prev_ExecutorEnd;
  407. ProcessUtility_hook = prev_ProcessUtility;
  408. }
  409. /*
  410. * shmem_startup hook: allocate or attach to shared memory,
  411. * then load any pre-existing statistics from file.
  412. * Also create and load the query-texts file, which is expected to exist
  413. * (even if empty) while the module is enabled.
  414. */
  415. static void
  416. pgss_shmem_startup(void)
  417. {
  418. bool found;
  419. HASHCTL info;
  420. FILE *file = NULL;
  421. FILE *qfile = NULL;
  422. uint32 header;
  423. int32 num;
  424. int32 pgver;
  425. int32 i;
  426. int buffer_size;
  427. char *buffer = NULL;
  428. if (prev_shmem_startup_hook)
  429. prev_shmem_startup_hook();
  430. /* reset in case this is a restart within the postmaster */
  431. pgss = NULL;
  432. pgss_hash = NULL;
  433. /*
  434. * Create or attach to the shared memory state, including hash table
  435. */
  436. LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE);
  437. pgss = ShmemInitStruct("pg_stat_statements",
  438. sizeof(pgssSharedState),
  439. &found);
  440. if (!found)
  441. {
  442. /* First time through ... */
  443. pgss->lock = &(GetNamedLWLockTranche("pg_stat_statements"))->lock;
  444. pgss->cur_median_usage = ASSUMED_MEDIAN_INIT;
  445. pgss->mean_query_len = ASSUMED_LENGTH_INIT;
  446. SpinLockInit(&pgss->mutex);
  447. pgss->extent = 0;
  448. pgss->n_writers = 0;
  449. pgss->gc_count = 0;
  450. }
  451. memset(&info, 0, sizeof(info));
  452. info.keysize = sizeof(pgssHashKey);
  453. info.entrysize = sizeof(pgssEntry);
  454. pgss_hash = ShmemInitHash("pg_stat_statements hash",
  455. pgss_max, pgss_max,
  456. &info,
  457. HASH_ELEM | HASH_BLOBS);
  458. LWLockRelease(AddinShmemInitLock);
  459. /*
  460. * If we're in the postmaster (or a standalone backend...), set up a shmem
  461. * exit hook to dump the statistics to disk.
  462. */
  463. if (!IsUnderPostmaster)
  464. on_shmem_exit(pgss_shmem_shutdown, (Datum) 0);
  465. /*
  466. * Done if some other process already completed our initialization.
  467. */
  468. if (found)
  469. return;
  470. /*
  471. * Note: we don't bother with locks here, because there should be no other
  472. * processes running when this code is reached.
  473. */
  474. /* Unlink query text file possibly left over from crash */
  475. unlink(PGSS_TEXT_FILE);
  476. /* Allocate new query text temp file */
  477. qfile = AllocateFile(PGSS_TEXT_FILE, PG_BINARY_W);
  478. if (qfile == NULL)
  479. goto write_error;
  480. /*
  481. * If we were told not to load old statistics, we're done. (Note we do
  482. * not try to unlink any old dump file in this case. This seems a bit
  483. * questionable but it's the historical behavior.)
  484. */
  485. if (!pgss_save)
  486. {
  487. FreeFile(qfile);
  488. return;
  489. }
  490. /*
  491. * Attempt to load old statistics from the dump file.
  492. */
  493. file = AllocateFile(PGSS_DUMP_FILE, PG_BINARY_R);
  494. if (file == NULL)
  495. {
  496. if (errno != ENOENT)
  497. goto read_error;
  498. /* No existing persisted stats file, so we're done */
  499. FreeFile(qfile);
  500. return;
  501. }
  502. buffer_size = 2048;
  503. buffer = (char *) palloc(buffer_size);
  504. if (fread(&header, sizeof(uint32), 1, file) != 1 ||
  505. fread(&pgver, sizeof(uint32), 1, file) != 1 ||
  506. fread(&num, sizeof(int32), 1, file) != 1)
  507. goto read_error;
  508. if (header != PGSS_FILE_HEADER ||
  509. pgver != PGSS_PG_MAJOR_VERSION)
  510. goto data_error;
  511. for (i = 0; i < num; i++)
  512. {
  513. pgssEntry temp;
  514. pgssEntry *entry;
  515. Size query_offset;
  516. if (fread(&temp, sizeof(pgssEntry), 1, file) != 1)
  517. goto read_error;
  518. /* Encoding is the only field we can easily sanity-check */
  519. if (!PG_VALID_BE_ENCODING(temp.encoding))
  520. goto data_error;
  521. /* Resize buffer as needed */
  522. if (temp.query_len >= buffer_size)
  523. {
  524. buffer_size = Max(buffer_size * 2, temp.query_len + 1);
  525. buffer = repalloc(buffer, buffer_size);
  526. }
  527. if (fread(buffer, 1, temp.query_len + 1, file) != temp.query_len + 1)
  528. goto read_error;
  529. /* Should have a trailing null, but let's make sure */
  530. buffer[temp.query_len] = '\0';
  531. /* Skip loading "sticky" entries */
  532. if (temp.counters.calls == 0)
  533. continue;
  534. /* Store the query text */
  535. query_offset = pgss->extent;
  536. if (fwrite(buffer, 1, temp.query_len + 1, qfile) != temp.query_len + 1)
  537. goto write_error;
  538. pgss->extent += temp.query_len + 1;
  539. /* make the hashtable entry (discards old entries if too many) */
  540. entry = entry_alloc(&temp.key, query_offset, temp.query_len,
  541. temp.encoding,
  542. false);
  543. /* copy in the actual stats */
  544. entry->counters = temp.counters;
  545. }
  546. pfree(buffer);
  547. FreeFile(file);
  548. FreeFile(qfile);
  549. /*
  550. * Remove the persisted stats file so it's not included in
  551. * backups/replication slaves, etc. A new file will be written on next
  552. * shutdown.
  553. *
  554. * Note: it's okay if the PGSS_TEXT_FILE is included in a basebackup,
  555. * because we remove that file on startup; it acts inversely to
  556. * PGSS_DUMP_FILE, in that it is only supposed to be around when the
  557. * server is running, whereas PGSS_DUMP_FILE is only supposed to be around
  558. * when the server is not running. Leaving the file creates no danger of
  559. * a newly restored database having a spurious record of execution costs,
  560. * which is what we're really concerned about here.
  561. */
  562. unlink(PGSS_DUMP_FILE);
  563. return;
  564. read_error:
  565. ereport(LOG,
  566. (errcode_for_file_access(),
  567. errmsg("could not read file \"%s\": %m",
  568. PGSS_DUMP_FILE)));
  569. goto fail;
  570. data_error:
  571. ereport(LOG,
  572. (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
  573. errmsg("ignoring invalid data in file \"%s\"",
  574. PGSS_DUMP_FILE)));
  575. goto fail;
  576. write_error:
  577. ereport(LOG,
  578. (errcode_for_file_access(),
  579. errmsg("could not write file \"%s\": %m",
  580. PGSS_TEXT_FILE)));
  581. fail:
  582. if (buffer)
  583. pfree(buffer);
  584. if (file)
  585. FreeFile(file);
  586. if (qfile)
  587. FreeFile(qfile);
  588. /* If possible, throw away the bogus file; ignore any error */
  589. unlink(PGSS_DUMP_FILE);
  590. /*
  591. * Don't unlink PGSS_TEXT_FILE here; it should always be around while the
  592. * server is running with pg_stat_statements enabled
  593. */
  594. }
  595. /*
  596. * shmem_shutdown hook: Dump statistics into file.
  597. *
  598. * Note: we don't bother with acquiring lock, because there should be no
  599. * other processes running when this is called.
  600. */
  601. static void
  602. pgss_shmem_shutdown(int code, Datum arg)
  603. {
  604. FILE *file;
  605. char *qbuffer = NULL;
  606. Size qbuffer_size = 0;
  607. HASH_SEQ_STATUS hash_seq;
  608. int32 num_entries;
  609. pgssEntry *entry;
  610. /* Don't try to dump during a crash. */
  611. if (code)
  612. return;
  613. /* Safety check ... shouldn't get here unless shmem is set up. */
  614. if (!pgss || !pgss_hash)
  615. return;
  616. /* Don't dump if told not to. */
  617. if (!pgss_save)
  618. return;
  619. file = AllocateFile(PGSS_DUMP_FILE ".tmp", PG_BINARY_W);
  620. if (file == NULL)
  621. goto error;
  622. if (fwrite(&PGSS_FILE_HEADER, sizeof(uint32), 1, file) != 1)
  623. goto error;
  624. if (fwrite(&PGSS_PG_MAJOR_VERSION, sizeof(uint32), 1, file) != 1)
  625. goto error;
  626. num_entries = hash_get_num_entries(pgss_hash);
  627. if (fwrite(&num_entries, sizeof(int32), 1, file) != 1)
  628. goto error;
  629. qbuffer = qtext_load_file(&qbuffer_size);
  630. if (qbuffer == NULL)
  631. goto error;
  632. /*
  633. * When serializing to disk, we store query texts immediately after their
  634. * entry data. Any orphaned query texts are thereby excluded.
  635. */
  636. hash_seq_init(&hash_seq, pgss_hash);
  637. while ((entry = hash_seq_search(&hash_seq)) != NULL)
  638. {
  639. int len = entry->query_len;
  640. char *qstr = qtext_fetch(entry->query_offset, len,
  641. qbuffer, qbuffer_size);
  642. if (qstr == NULL)
  643. continue; /* Ignore any entries with bogus texts */
  644. if (fwrite(entry, sizeof(pgssEntry), 1, file) != 1 ||
  645. fwrite(qstr, 1, len + 1, file) != len + 1)
  646. {
  647. /* note: we assume hash_seq_term won't change errno */
  648. hash_seq_term(&hash_seq);
  649. goto error;
  650. }
  651. }
  652. free(qbuffer);
  653. qbuffer = NULL;
  654. if (FreeFile(file))
  655. {
  656. file = NULL;
  657. goto error;
  658. }
  659. /*
  660. * Rename file into place, so we atomically replace any old one.
  661. */
  662. (void) durable_rename(PGSS_DUMP_FILE ".tmp", PGSS_DUMP_FILE, LOG);
  663. /* Unlink query-texts file; it's not needed while shutdown */
  664. unlink(PGSS_TEXT_FILE);
  665. return;
  666. error:
  667. ereport(LOG,
  668. (errcode_for_file_access(),
  669. errmsg("could not write file \"%s\": %m",
  670. PGSS_DUMP_FILE ".tmp")));
  671. if (qbuffer)
  672. free(qbuffer);
  673. if (file)
  674. FreeFile(file);
  675. unlink(PGSS_DUMP_FILE ".tmp");
  676. unlink(PGSS_TEXT_FILE);
  677. }
  678. /*
  679. * Post-parse-analysis hook: mark query with a queryId
  680. */
  681. static void
  682. pgss_post_parse_analyze(ParseState *pstate, Query *query)
  683. {
  684. pgssJumbleState jstate;
  685. if (prev_post_parse_analyze_hook)
  686. prev_post_parse_analyze_hook(pstate, query);
  687. /* Assert we didn't do this already */
  688. Assert(query->queryId == UINT64CONST(0));
  689. /* Safety check... */
  690. if (!pgss || !pgss_hash)
  691. return;
  692. /*
  693. * Utility statements get queryId zero. We do this even in cases where
  694. * the statement contains an optimizable statement for which a queryId
  695. * could be derived (such as EXPLAIN or DECLARE CURSOR). For such cases,
  696. * runtime control will first go through ProcessUtility and then the
  697. * executor, and we don't want the executor hooks to do anything, since we
  698. * are already measuring the statement's costs at the utility level.
  699. */
  700. if (query->utilityStmt)
  701. {
  702. query->queryId = UINT64CONST(0);
  703. return;
  704. }
  705. /* Set up workspace for query jumbling */
  706. jstate.jumble = (unsigned char *) palloc(JUMBLE_SIZE);
  707. jstate.jumble_len = 0;
  708. jstate.clocations_buf_size = 32;
  709. jstate.clocations = (pgssLocationLen *)
  710. palloc(jstate.clocations_buf_size * sizeof(pgssLocationLen));
  711. jstate.clocations_count = 0;
  712. jstate.highest_extern_param_id = 0;
  713. /* Compute query ID and mark the Query node with it */
  714. JumbleQuery(&jstate, query);
  715. query->queryId =
  716. DatumGetUInt64(hash_any_extended(jstate.jumble, jstate.jumble_len, 0));
  717. /*
  718. * If we are unlucky enough to get a hash of zero, use 1 instead, to
  719. * prevent confusion with the utility-statement case.
  720. */
  721. if (query->queryId == UINT64CONST(0))
  722. query->queryId = UINT64CONST(1);
  723. /*
  724. * If we were able to identify any ignorable constants, we immediately
  725. * create a hash table entry for the query, so that we can record the
  726. * normalized form of the query string. If there were no such constants,
  727. * the normalized string would be the same as the query text anyway, so
  728. * there's no need for an early entry.
  729. */
  730. if (jstate.clocations_count > 0)
  731. pgss_store(pstate->p_sourcetext,
  732. query->queryId,
  733. query->stmt_location,
  734. query->stmt_len,
  735. 0,
  736. 0,
  737. NULL,
  738. &jstate);
  739. }
  740. /*
  741. * ExecutorStart hook: start up tracking if needed
  742. */
  743. static void
  744. pgss_ExecutorStart(QueryDesc *queryDesc, int eflags)
  745. {
  746. if (prev_ExecutorStart)
  747. prev_ExecutorStart(queryDesc, eflags);
  748. else
  749. standard_ExecutorStart(queryDesc, eflags);
  750. /*
  751. * If query has queryId zero, don't track it. This prevents double
  752. * counting of optimizable statements that are directly contained in
  753. * utility statements.
  754. */
  755. if (pgss_enabled() && queryDesc->plannedstmt->queryId != UINT64CONST(0))
  756. {
  757. /*
  758. * Set up to track total elapsed time in ExecutorRun. Make sure the
  759. * space is allocated in the per-query context so it will go away at
  760. * ExecutorEnd.
  761. */
  762. if (queryDesc->totaltime == NULL)
  763. {
  764. MemoryContext oldcxt;
  765. oldcxt = MemoryContextSwitchTo(queryDesc->estate->es_query_cxt);
  766. queryDesc->totaltime = InstrAlloc(1, INSTRUMENT_ALL);
  767. MemoryContextSwitchTo(oldcxt);
  768. }
  769. }
  770. }
  771. /*
  772. * ExecutorRun hook: all we need do is track nesting depth
  773. */
  774. static void
  775. pgss_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count,
  776. bool execute_once)
  777. {
  778. nested_level++;
  779. PG_TRY();
  780. {
  781. if (prev_ExecutorRun)
  782. prev_ExecutorRun(queryDesc, direction, count, execute_once);
  783. else
  784. standard_ExecutorRun(queryDesc, direction, count, execute_once);
  785. nested_level--;
  786. }
  787. PG_CATCH();
  788. {
  789. nested_level--;
  790. PG_RE_THROW();
  791. }
  792. PG_END_TRY();
  793. }
  794. /*
  795. * ExecutorFinish hook: all we need do is track nesting depth
  796. */
  797. static void
  798. pgss_ExecutorFinish(QueryDesc *queryDesc)
  799. {
  800. nested_level++;
  801. PG_TRY();
  802. {
  803. if (prev_ExecutorFinish)
  804. prev_ExecutorFinish(queryDesc);
  805. else
  806. standard_ExecutorFinish(queryDesc);
  807. nested_level--;
  808. }
  809. PG_CATCH();
  810. {
  811. nested_level--;
  812. PG_RE_THROW();
  813. }
  814. PG_END_TRY();
  815. }
  816. /*
  817. * ExecutorEnd hook: store results if needed
  818. */
  819. static void
  820. pgss_ExecutorEnd(QueryDesc *queryDesc)
  821. {
  822. uint64 queryId = queryDesc->plannedstmt->queryId;
  823. if (queryId != UINT64CONST(0) && queryDesc->totaltime && pgss_enabled())
  824. {
  825. /*
  826. * Make sure stats accumulation is done. (Note: it's okay if several
  827. * levels of hook all do this.)
  828. */
  829. InstrEndLoop(queryDesc->totaltime);
  830. pgss_store(queryDesc->sourceText,
  831. queryId,
  832. queryDesc->plannedstmt->stmt_location,
  833. queryDesc->plannedstmt->stmt_len,
  834. queryDesc->totaltime->total * 1000.0, /* convert to msec */
  835. queryDesc->estate->es_processed,
  836. &queryDesc->totaltime->bufusage,
  837. NULL);
  838. }
  839. if (prev_ExecutorEnd)
  840. prev_ExecutorEnd(queryDesc);
  841. else
  842. standard_ExecutorEnd(queryDesc);
  843. }
  844. /*
  845. * ProcessUtility hook
  846. */
  847. static void
  848. pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString,
  849. ProcessUtilityContext context,
  850. ParamListInfo params, QueryEnvironment *queryEnv,
  851. DestReceiver *dest, char *completionTag)
  852. {
  853. Node *parsetree = pstmt->utilityStmt;
  854. /*
  855. * If it's an EXECUTE statement, we don't track it and don't increment the
  856. * nesting level. This allows the cycles to be charged to the underlying
  857. * PREPARE instead (by the Executor hooks), which is much more useful.
  858. *
  859. * We also don't track execution of PREPARE. If we did, we would get one
  860. * hash table entry for the PREPARE (with hash calculated from the query
  861. * string), and then a different one with the same query string (but hash
  862. * calculated from the query tree) would be used to accumulate costs of
  863. * ensuing EXECUTEs. This would be confusing, and inconsistent with other
  864. * cases where planning time is not included at all.
  865. *
  866. * Likewise, we don't track execution of DEALLOCATE.
  867. */
  868. if (pgss_track_utility && pgss_enabled() &&
  869. !IsA(parsetree, ExecuteStmt) &&
  870. !IsA(parsetree, PrepareStmt) &&
  871. !IsA(parsetree, DeallocateStmt))
  872. {
  873. instr_time start;
  874. instr_time duration;
  875. uint64 rows;
  876. BufferUsage bufusage_start,
  877. bufusage;
  878. bufusage_start = pgBufferUsage;
  879. INSTR_TIME_SET_CURRENT(start);
  880. nested_level++;
  881. PG_TRY();
  882. {
  883. if (prev_ProcessUtility)
  884. prev_ProcessUtility(pstmt, queryString,
  885. context, params, queryEnv,
  886. dest, completionTag);
  887. else
  888. standard_ProcessUtility(pstmt, queryString,
  889. context, params, queryEnv,
  890. dest, completionTag);
  891. nested_level--;
  892. }
  893. PG_CATCH();
  894. {
  895. nested_level--;
  896. PG_RE_THROW();
  897. }
  898. PG_END_TRY();
  899. INSTR_TIME_SET_CURRENT(duration);
  900. INSTR_TIME_SUBTRACT(duration, start);
  901. /* parse command tag to retrieve the number of affected rows. */
  902. if (completionTag &&
  903. strncmp(completionTag, "COPY ", 5) == 0)
  904. rows = pg_strtouint64(completionTag + 5, NULL, 10);
  905. else
  906. rows = 0;
  907. /* calc differences of buffer counters. */
  908. bufusage.shared_blks_hit =
  909. pgBufferUsage.shared_blks_hit - bufusage_start.shared_blks_hit;
  910. bufusage.shared_blks_read =
  911. pgBufferUsage.shared_blks_read - bufusage_start.shared_blks_read;
  912. bufusage.shared_blks_dirtied =
  913. pgBufferUsage.shared_blks_dirtied - bufusage_start.shared_blks_dirtied;
  914. bufusage.shared_blks_written =
  915. pgBufferUsage.shared_blks_written - bufusage_start.shared_blks_written;
  916. bufusage.local_blks_hit =
  917. pgBufferUsage.local_blks_hit - bufusage_start.local_blks_hit;
  918. bufusage.local_blks_read =
  919. pgBufferUsage.local_blks_read - bufusage_start.local_blks_read;
  920. bufusage.local_blks_dirtied =
  921. pgBufferUsage.local_blks_dirtied - bufusage_start.local_blks_dirtied;
  922. bufusage.local_blks_written =
  923. pgBufferUsage.local_blks_written - bufusage_start.local_blks_written;
  924. bufusage.temp_blks_read =
  925. pgBufferUsage.temp_blks_read - bufusage_start.temp_blks_read;
  926. bufusage.temp_blks_written =
  927. pgBufferUsage.temp_blks_written - bufusage_start.temp_blks_written;
  928. bufusage.blk_read_time = pgBufferUsage.blk_read_time;
  929. INSTR_TIME_SUBTRACT(bufusage.blk_read_time, bufusage_start.blk_read_time);
  930. bufusage.blk_write_time = pgBufferUsage.blk_write_time;
  931. INSTR_TIME_SUBTRACT(bufusage.blk_write_time, bufusage_start.blk_write_time);
  932. pgss_store(queryString,
  933. 0, /* signal that it's a utility stmt */
  934. pstmt->stmt_location,
  935. pstmt->stmt_len,
  936. INSTR_TIME_GET_MILLISEC(duration),
  937. rows,
  938. &bufusage,
  939. NULL);
  940. }
  941. else
  942. {
  943. if (prev_ProcessUtility)
  944. prev_ProcessUtility(pstmt, queryString,
  945. context, params, queryEnv,
  946. dest, completionTag);
  947. else
  948. standard_ProcessUtility(pstmt, queryString,
  949. context, params, queryEnv,
  950. dest, completionTag);
  951. }
  952. }
  953. /*
  954. * Given an arbitrarily long query string, produce a hash for the purposes of
  955. * identifying the query, without normalizing constants. Used when hashing
  956. * utility statements.
  957. */
  958. static uint64
  959. pgss_hash_string(const char *str, int len)
  960. {
  961. return DatumGetUInt64(hash_any_extended((const unsigned char *) str,
  962. len, 0));
  963. }
  964. /*
  965. * Store some statistics for a statement.
  966. *
  967. * If queryId is 0 then this is a utility statement and we should compute
  968. * a suitable queryId internally.
  969. *
  970. * If jstate is not NULL then we're trying to create an entry for which
  971. * we have no statistics as yet; we just want to record the normalized
  972. * query string. total_time, rows, bufusage are ignored in this case.
  973. */
  974. static void
  975. pgss_store(const char *query, uint64 queryId,
  976. int query_location, int query_len,
  977. double total_time, uint64 rows,
  978. const BufferUsage *bufusage,
  979. pgssJumbleState *jstate)
  980. {
  981. pgssHashKey key;
  982. pgssEntry *entry;
  983. char *norm_query = NULL;
  984. int encoding = GetDatabaseEncoding();
  985. Assert(query != NULL);
  986. /* Safety check... */
  987. if (!pgss || !pgss_hash)
  988. return;
  989. /*
  990. * Confine our attention to the relevant part of the string, if the query
  991. * is a portion of a multi-statement source string.
  992. *
  993. * First apply starting offset, unless it's -1 (unknown).
  994. */
  995. if (query_location >= 0)
  996. {
  997. Assert(query_location <= strlen(query));
  998. query += query_location;
  999. /* Length of 0 (or -1) means "rest of string" */
  1000. if (query_len <= 0)
  1001. query_len = strlen(query);
  1002. else
  1003. Assert(query_len <= strlen(query));
  1004. }
  1005. else
  1006. {
  1007. /* If query location is unknown, distrust query_len as well */
  1008. query_location = 0;
  1009. query_len = strlen(query);
  1010. }
  1011. /*
  1012. * Discard leading and trailing whitespace, too. Use scanner_isspace()
  1013. * not libc's isspace(), because we want to match the lexer's behavior.
  1014. */
  1015. while (query_len > 0 && scanner_isspace(query[0]))
  1016. query++, query_location++, query_len--;
  1017. while (query_len > 0 && scanner_isspace(query[query_len - 1]))
  1018. query_len--;
  1019. /*
  1020. * For utility statements, we just hash the query string to get an ID.
  1021. */
  1022. if (queryId == UINT64CONST(0))
  1023. queryId = pgss_hash_string(query, query_len);
  1024. /* Set up key for hashtable search */
  1025. key.userid = GetUserId();
  1026. key.dbid = MyDatabaseId;
  1027. key.queryid = queryId;
  1028. /* Lookup the hash table entry with shared lock. */
  1029. LWLockAcquire(pgss->lock, LW_SHARED);
  1030. entry = (pgssEntry *) hash_search(pgss_hash, &key, HASH_FIND, NULL);
  1031. /* Create new entry, if not present */
  1032. if (!entry)
  1033. {
  1034. Size query_offset;
  1035. int gc_count;
  1036. bool stored;
  1037. bool do_gc;
  1038. /*
  1039. * Create a new, normalized query string if caller asked. We don't
  1040. * need to hold the lock while doing this work. (Note: in any case,
  1041. * it's possible that someone else creates a duplicate hashtable entry
  1042. * in the interval where we don't hold the lock below. That case is
  1043. * handled by entry_alloc.)
  1044. */
  1045. if (jstate)
  1046. {
  1047. LWLockRelease(pgss->lock);
  1048. norm_query = generate_normalized_query(jstate, query,
  1049. query_location,
  1050. &query_len,
  1051. encoding);
  1052. LWLockAcquire(pgss->lock, LW_SHARED);
  1053. }
  1054. /* Append new query text to file with only shared lock held */
  1055. stored = qtext_store(norm_query ? norm_query : query, query_len,
  1056. &query_offset, &gc_count);
  1057. /*
  1058. * Determine whether we need to garbage collect external query texts
  1059. * while the shared lock is still held. This micro-optimization
  1060. * avoids taking the time to decide this while holding exclusive lock.
  1061. */
  1062. do_gc = need_gc_qtexts();
  1063. /* Need exclusive lock to make a new hashtable entry - promote */
  1064. LWLockRelease(pgss->lock);
  1065. LWLockAcquire(pgss->lock, LW_EXCLUSIVE);
  1066. /*
  1067. * A garbage collection may have occurred while we weren't holding the
  1068. * lock. In the unlikely event that this happens, the query text we
  1069. * stored above will have been garbage collected, so write it again.
  1070. * This should be infrequent enough that doing it while holding
  1071. * exclusive lock isn't a performance problem.
  1072. */
  1073. if (!stored || pgss->gc_count != gc_count)
  1074. stored = qtext_store(norm_query ? norm_query : query, query_len,
  1075. &query_offset, NULL);
  1076. /* If we failed to write to the text file, give up */
  1077. if (!stored)
  1078. goto done;
  1079. /* OK to create a new hashtable entry */
  1080. entry = entry_alloc(&key, query_offset, query_len, encoding,
  1081. jstate != NULL);
  1082. /* If needed, perform garbage collection while exclusive lock held */
  1083. if (do_gc)
  1084. gc_qtexts();
  1085. }
  1086. /* Increment the counts, except when jstate is not NULL */
  1087. if (!jstate)
  1088. {
  1089. /*
  1090. * Grab the spinlock while updating the counters (see comment about
  1091. * locking rules at the head of the file)
  1092. */
  1093. volatile pgssEntry *e = (volatile pgssEntry *) entry;
  1094. SpinLockAcquire(&e->mutex);
  1095. /* "Unstick" entry if it was previously sticky */
  1096. if (e->counters.calls == 0)
  1097. e->counters.usage = USAGE_INIT;
  1098. e->counters.calls += 1;
  1099. e->counters.total_time += total_time;
  1100. if (e->counters.calls == 1)
  1101. {
  1102. e->counters.min_time = total_time;
  1103. e->counters.max_time = total_time;
  1104. e->counters.mean_time = total_time;
  1105. }
  1106. else
  1107. {
  1108. /*
  1109. * Welford's method for accurately computing variance. See
  1110. * <http://www.johndcook.com/blog/standard_deviation/>
  1111. */
  1112. double old_mean = e->counters.mean_time;
  1113. e->counters.mean_time +=
  1114. (total_time - old_mean) / e->counters.calls;
  1115. e->counters.sum_var_time +=
  1116. (total_time - old_mean) * (total_time - e->counters.mean_time);
  1117. /* calculate min and max time */
  1118. if (e->counters.min_time > total_time)
  1119. e->counters.min_time = total_time;
  1120. if (e->counters.max_time < total_time)
  1121. e->counters.max_time = total_time;
  1122. }
  1123. e->counters.rows += rows;
  1124. e->counters.shared_blks_hit += bufusage->shared_blks_hit;
  1125. e->counters.shared_blks_read += bufusage->shared_blks_read;
  1126. e->counters.shared_blks_dirtied += bufusage->shared_blks_dirtied;
  1127. e->counters.shared_blks_written += bufusage->shared_blks_written;
  1128. e->counters.local_blks_hit += bufusage->local_blks_hit;
  1129. e->counters.local_blks_read += bufusage->local_blks_read;
  1130. e->counters.local_blks_dirtied += bufusage->local_blks_dirtied;
  1131. e->counters.local_blks_written += bufusage->local_blks_written;
  1132. e->counters.temp_blks_read += bufusage->temp_blks_read;
  1133. e->counters.temp_blks_written += bufusage->temp_blks_written;
  1134. e->counters.blk_read_time += INSTR_TIME_GET_MILLISEC(bufusage->blk_read_time);
  1135. e->counters.blk_write_time += INSTR_TIME_GET_MILLISEC(bufusage->blk_write_time);
  1136. e->counters.usage += USAGE_EXEC(total_time);
  1137. SpinLockRelease(&e->mutex);
  1138. }
  1139. done:
  1140. LWLockRelease(pgss->lock);
  1141. /* We postpone this clean-up until we're out of the lock */
  1142. if (norm_query)
  1143. pfree(norm_query);
  1144. }
  1145. /*
  1146. * Reset all statement statistics.
  1147. */
  1148. Datum
  1149. pg_stat_statements_reset(PG_FUNCTION_ARGS)
  1150. {
  1151. if (!pgss || !pgss_hash)
  1152. ereport(ERROR,
  1153. (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
  1154. errmsg("pg_stat_statements must be loaded via shared_preload_libraries")));
  1155. entry_reset();
  1156. PG_RETURN_VOID();
  1157. }
  1158. /* Number of output arguments (columns) for various API versions */
  1159. #define PG_STAT_STATEMENTS_COLS_V1_0 14
  1160. #define PG_STAT_STATEMENTS_COLS_V1_1 18
  1161. #define PG_STAT_STATEMENTS_COLS_V1_2 19
  1162. #define PG_STAT_STATEMENTS_COLS_V1_3 23
  1163. #define PG_STAT_STATEMENTS_COLS 23 /* maximum of above */
  1164. /*
  1165. * Retrieve statement statistics.
  1166. *
  1167. * The SQL API of this function has changed multiple times, and will likely
  1168. * do so again in future. To support the case where a newer version of this
  1169. * loadable module is being used with an old SQL declaration of the function,
  1170. * we continue to support the older API versions. For 1.2 and later, the
  1171. * expected API version is identified by embedding it in the C name of the
  1172. * function. Unfortunately we weren't bright enough to do that for 1.1.
  1173. */
  1174. Datum
  1175. pg_stat_statements_1_3(PG_FUNCTION_ARGS)
  1176. {
  1177. bool showtext = PG_GETARG_BOOL(0);
  1178. pg_stat_statements_internal(fcinfo, PGSS_V1_3, showtext);
  1179. return (Datum) 0;
  1180. }
  1181. Datum
  1182. pg_stat_statements_1_2(PG_FUNCTION_ARGS)
  1183. {
  1184. bool showtext = PG_GETARG_BOOL(0);
  1185. pg_stat_statements_internal(fcinfo, PGSS_V1_2, showtext);
  1186. return (Datum) 0;
  1187. }
  1188. /*
  1189. * Legacy entry point for pg_stat_statements() API versions 1.0 and 1.1.
  1190. * This can be removed someday, perhaps.
  1191. */
  1192. Datum
  1193. pg_stat_statements(PG_FUNCTION_ARGS)
  1194. {
  1195. /* If it's really API 1.1, we'll figure that out below */
  1196. pg_stat_statements_internal(fcinfo, PGSS_V1_0, true);
  1197. return (Datum) 0;
  1198. }
  1199. /* Common code for all versions of pg_stat_statements() */
  1200. static void
  1201. pg_stat_statements_internal(FunctionCallInfo fcinfo,
  1202. pgssVersion api_version,
  1203. bool showtext)
  1204. {
  1205. ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
  1206. TupleDesc tupdesc;
  1207. Tuplestorestate *tupstore;
  1208. MemoryContext per_query_ctx;
  1209. MemoryContext oldcontext;
  1210. Oid userid = GetUserId();
  1211. bool is_allowed_role = false;
  1212. char *qbuffer = NULL;
  1213. Size qbuffer_size = 0;
  1214. Size extent = 0;
  1215. int gc_count = 0;
  1216. HASH_SEQ_STATUS hash_seq;
  1217. pgssEntry *entry;
  1218. /* Superusers or members of pg_read_all_stats members are allowed */
  1219. is_allowed_role = is_member_of_role(GetUserId(), DEFAULT_ROLE_READ_ALL_STATS);
  1220. /* hash table must exist already */
  1221. if (!pgss || !pgss_hash)
  1222. ereport(ERROR,
  1223. (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
  1224. errmsg("pg_stat_statements must be loaded via shared_preload_libraries")));
  1225. /* check to see if caller supports us returning a tuplestore */
  1226. if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
  1227. ereport(ERROR,
  1228. (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
  1229. errmsg("set-valued function called in context that cannot accept a set")));
  1230. if (!(rsinfo->allowedModes & SFRM_Materialize))
  1231. ereport(ERROR,
  1232. (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
  1233. errmsg("materialize mode required, but it is not " \
  1234. "allowed in this context")));
  1235. /* Switch into long-lived context to construct returned data structures */
  1236. per_query_ctx = rsinfo->econtext->ecxt_per_query_memory;
  1237. oldcontext = MemoryContextSwitchTo(per_query_ctx);
  1238. /* Build a tuple descriptor for our result type */
  1239. if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
  1240. elog(ERROR, "return type must be a row type");
  1241. /*
  1242. * Check we have the expected number of output arguments. Aside from
  1243. * being a good safety check, we need a kluge here to detect API version
  1244. * 1.1, which was wedged into the code in an ill-considered way.
  1245. */
  1246. switch (tupdesc->natts)
  1247. {
  1248. case PG_STAT_STATEMENTS_COLS_V1_0:
  1249. if (api_version != PGSS_V1_0)
  1250. elog(ERROR, "incorrect number of output arguments");
  1251. break;
  1252. case PG_STAT_STATEMENTS_COLS_V1_1:
  1253. /* pg_stat_statements() should have told us 1.0 */
  1254. if (api_version != PGSS_V1_0)
  1255. elog(ERROR, "incorrect number of output arguments");
  1256. api_version = PGSS_V1_1;
  1257. break;
  1258. case PG_STAT_STATEMENTS_COLS_V1_2:
  1259. if (api_version != PGSS_V1_2)
  1260. elog(ERROR, "incorrect number of output arguments");
  1261. break;
  1262. case PG_STAT_STATEMENTS_COLS_V1_3:
  1263. if (api_version != PGSS_V1_3)
  1264. elog(ERROR, "incorrect number of output arguments");
  1265. break;
  1266. default:
  1267. elog(ERROR, "incorrect number of output arguments");
  1268. }
  1269. tupstore = tuplestore_begin_heap(true, false, work_mem);
  1270. rsinfo->returnMode = SFRM_Materialize;
  1271. rsinfo->setResult = tupstore;
  1272. rsinfo->setDesc = tupdesc;
  1273. MemoryContextSwitchTo(oldcontext);
  1274. /*
  1275. * We'd like to load the query text file (if needed) while not holding any
  1276. * lock on pgss->lock. In the worst case we'll have to do this again
  1277. * after we have the lock, but it's unlikely enough to make this a win
  1278. * despite occasional duplicated work. We need to reload if anybody
  1279. * writes to the file (either a retail qtext_store(), or a garbage
  1280. * collection) between this point and where we've gotten shared lock. If
  1281. * a qtext_store is actually in progress when we look, we might as well
  1282. * skip the speculative load entirely.
  1283. */
  1284. if (showtext)
  1285. {
  1286. int n_writers;
  1287. /* Take the mutex so we can examine variables */
  1288. {
  1289. volatile pgssSharedState *s = (volatile pgssSharedState *) pgss;
  1290. SpinLockAcquire(&s->mutex);
  1291. extent = s->extent;
  1292. n_writers = s->n_writers;
  1293. gc_count = s->gc_count;
  1294. SpinLockRelease(&s->mutex);
  1295. }
  1296. /* No point in loading file now if there are active writers */
  1297. if (n_writers == 0)
  1298. qbuffer = qtext_load_file(&qbuffer_size);
  1299. }
  1300. /*
  1301. * Get shared lock, load or reload the query text file if we must, and
  1302. * iterate over the hashtable entries.
  1303. *
  1304. * With a large hash table, we might be holding the lock rather longer
  1305. * than one could wish. However, this only blocks creation of new hash
  1306. * table entries, and the larger the hash table the less likely that is to
  1307. * be needed. So we can hope this is okay. Perhaps someday we'll decide
  1308. * we need to partition the hash table to limit the time spent holding any
  1309. * one lock.
  1310. */
  1311. LWLockAcquire(pgss->lock, LW_SHARED);
  1312. if (showtext)
  1313. {
  1314. /*
  1315. * Here it is safe to examine extent and gc_count without taking the
  1316. * mutex. Note that although other processes might change
  1317. * pgss->extent just after we look at it, the strings they then write
  1318. * into the file cannot yet be referenced in the hashtable, so we
  1319. * don't care whether we see them or not.
  1320. *
  1321. * If qtext_load_file fails, we just press on; we'll return NULL for
  1322. * every query text.
  1323. */
  1324. if (qbuffer == NULL ||
  1325. pgss->extent != extent ||
  1326. pgss->gc_count != gc_count)
  1327. {
  1328. if (qbuffer)
  1329. free(qbuffer);
  1330. qbuffer = qtext_load_file(&qbuffer_size);
  1331. }
  1332. }
  1333. hash_seq_init(&hash_seq, pgss_hash);
  1334. while ((entry = hash_seq_search(&hash_seq)) != NULL)
  1335. {
  1336. Datum values[PG_STAT_STATEMENTS_COLS];
  1337. bool nulls[PG_STAT_STATEMENTS_COLS];
  1338. int i = 0;
  1339. Counters tmp;
  1340. double stddev;
  1341. int64 queryid = entry->key.queryid;
  1342. memset(values, 0, sizeof(values));
  1343. memset(nulls, 0, sizeof(nulls));
  1344. values[i++] = ObjectIdGetDatum(entry->key.userid);
  1345. values[i++] = ObjectIdGetDatum(entry->key.dbid);
  1346. if (is_allowed_role || entry->key.userid == userid)
  1347. {
  1348. if (api_version >= PGSS_V1_2)
  1349. values[i++] = Int64GetDatumFast(queryid);
  1350. if (showtext)
  1351. {
  1352. char *qstr = qtext_fetch(entry->query_offset,
  1353. entry->query_len,
  1354. qbuffer,
  1355. qbuffer_size);
  1356. if (qstr)
  1357. {
  1358. char *enc;
  1359. enc = pg_any_to_server(qstr,
  1360. entry->query_len,
  1361. entry->encoding);
  1362. values[i++] = CStringGetTextDatum(enc);
  1363. if (enc != qstr)
  1364. pfree(enc);
  1365. }
  1366. else
  1367. {
  1368. /* Just return a null if we fail to find the text */
  1369. nulls[i++] = true;
  1370. }
  1371. }
  1372. else
  1373. {
  1374. /* Query text not requested */
  1375. nulls[i++] = true;
  1376. }
  1377. }
  1378. else
  1379. {
  1380. /* Don't show queryid */
  1381. if (api_version >= PGSS_V1_2)
  1382. nulls[i++] = true;
  1383. /*
  1384. * Don't show query text, but hint as to the reason for not doing
  1385. * so if it was requested
  1386. */
  1387. if (showtext)
  1388. values[i++] = CStringGetTextDatum("<insufficient privilege>");
  1389. else
  1390. nulls[i++] = true;
  1391. }
  1392. /* copy counters to a local variable to keep locking time short */
  1393. {
  1394. volatile pgssEntry *e = (volatile pgssEntry *) entry;
  1395. SpinLockAcquire(&e->mutex);
  1396. tmp = e->counters;
  1397. SpinLockRelease(&e->mutex);
  1398. }
  1399. /* Skip entry if unexecuted (ie, it's a pending "sticky" entry) */
  1400. if (tmp.calls == 0)
  1401. continue;
  1402. values[i++] = Int64GetDatumFast(tmp.calls);
  1403. values[i++] = Float8GetDatumFast(tmp.total_time);
  1404. if (api_version >= PGSS_V1_3)
  1405. {
  1406. values[i++] = Float8GetDatumFast(tmp.min_time);
  1407. values[i++] = Float8GetDatumFast(tmp.max_time);
  1408. values[i++] = Float8GetDatumFast(tmp.mean_time);
  1409. /*
  1410. * Note we are calculating the population variance here, not the
  1411. * sample variance, as we have data for the whole population, so
  1412. * Bessel's correction is not used, and we don't divide by
  1413. * tmp.calls - 1.
  1414. */
  1415. if (tmp.calls > 1)
  1416. stddev = sqrt(tmp.sum_var_time / tmp.calls);
  1417. else
  1418. stddev = 0.0;
  1419. values[i++] = Float8GetDatumFast(stddev);
  1420. }
  1421. values[i++] = Int64GetDatumFast(tmp.rows);
  1422. values[i++] = Int64GetDatumFast(tmp.shared_blks_hit);
  1423. values[i++] = Int64GetDatumFast(tmp.shared_blks_read);
  1424. if (api_version >= PGSS_V1_1)
  1425. values[i++] = Int64GetDatumFast(tmp.shared_blks_dirtied);
  1426. values[i++] = Int64GetDatumFast(tmp.shared_blks_written);
  1427. values[i++] = Int64GetDatumFast(tmp.local_blks_hit);
  1428. values[i++] = Int64GetDatumFast(tmp.local_blks_read);
  1429. if (api_version >= PGSS_V1_1)
  1430. values[i++] = Int64GetDatumFast(tmp.local_blks_dirtied);
  1431. values[i++] = Int64GetDatumFast(tmp.local_blks_written);
  1432. values[i++] = Int64GetDatumFast(tmp.temp_blks_read);
  1433. values[i++] = Int64GetDatumFast(tmp.temp_blks_written);
  1434. if (api_version >= PGSS_V1_1)
  1435. {
  1436. values[i++] = Float8GetDatumFast(tmp.blk_read_time);
  1437. values[i++] = Float8GetDatumFast(tmp.blk_write_time);
  1438. }
  1439. Assert(i == (api_version == PGSS_V1_0 ? PG_STAT_STATEMENTS_COLS_V1_0 :
  1440. api_version == PGSS_V1_1 ? PG_STAT_STATEMENTS_COLS_V1_1 :
  1441. api_version == PGSS_V1_2 ? PG_STAT_STATEMENTS_COLS_V1_2 :
  1442. api_version == PGSS_V1_3 ? PG_STAT_STATEMENTS_COLS_V1_3 :
  1443. -1 /* fail if you forget to update this assert */ ));
  1444. tuplestore_putvalues(tupstore, tupdesc, values, nulls);
  1445. }
  1446. /* clean up and return the tuplestore */
  1447. LWLockRelease(pgss->lock);
  1448. if (qbuffer)
  1449. free(qbuffer);
  1450. tuplestore_donestoring(tupstore);
  1451. }
  1452. /*
  1453. * Estimate shared memory space needed.
  1454. */
  1455. static Size
  1456. pgss_memsize(void)
  1457. {
  1458. Size size;
  1459. size = MAXALIGN(sizeof(pgssSharedState));
  1460. size = add_size(size, hash_estimate_size(pgss_max, sizeof(pgssEntry)));
  1461. return size;
  1462. }
  1463. /*
  1464. * Allocate a new hashtable entry.
  1465. * caller must hold an exclusive lock on pgss->lock
  1466. *
  1467. * "query" need not be null-terminated; we rely on query_len instead
  1468. *
  1469. * If "sticky" is true, make the new entry artificially sticky so that it will
  1470. * probably still be there when the query finishes execution. We do this by
  1471. * giving it a median usage value rather than the normal value. (Strictly
  1472. * speaking, query strings are normalized on a best effort basis, though it
  1473. * would be difficult to demonstrate this even under artificial conditions.)
  1474. *
  1475. * Note: despite needing exclusive lock, it's not an error for the target
  1476. * entry to already exist. This is because pgss_store releases and
  1477. * reacquires lock after failing to find a match; so someone else could
  1478. * have made the entry while we waited to get exclusive lock.
  1479. */
  1480. static pgssEntry *
  1481. entry_alloc(pgssHashKey *key, Size query_offset, int query_len, int encoding,
  1482. bool sticky)
  1483. {
  1484. pgssEntry *entry;
  1485. bool found;
  1486. /* Make space if needed */
  1487. while (hash_get_num_entries(pgss_hash) >= pgss_max)
  1488. entry_dealloc();
  1489. /* Find or create an entry with desired hash code */
  1490. entry = (pgssEntry *) hash_search(pgss_hash, key, HASH_ENTER, &found);
  1491. if (!found)
  1492. {
  1493. /* New entry, initialize it */
  1494. /* reset the statistics */
  1495. memset(&entry->counters, 0, sizeof(Counters));
  1496. /* set the appropriate initial usage count */
  1497. entry->counters.usage = sticky ? pgss->cur_median_usage : USAGE_INIT;
  1498. /* re-initialize the mutex each time ... we assume no one using it */
  1499. SpinLockInit(&entry->mutex);
  1500. /* ... and don't forget the query text metadata */
  1501. Assert(query_len >= 0);
  1502. entry->query_offset = query_offset;
  1503. entry->query_len = query_len;
  1504. entry->encoding = encoding;
  1505. }
  1506. return entry;
  1507. }
  1508. /*
  1509. * qsort comparator for sorting into increasing usage order
  1510. */
  1511. static int
  1512. entry_cmp(const void *lhs, const void *rhs)
  1513. {
  1514. double l_usage = (*(pgssEntry *const *) lhs)->counters.usage;
  1515. double r_usage = (*(pgssEntry *const *) rhs)->counters.usage;
  1516. if (l_usage < r_usage)
  1517. return -1;
  1518. else if (l_usage > r_usage)
  1519. return +1;
  1520. else
  1521. return 0;
  1522. }
  1523. /*
  1524. * Deallocate least-used entries.
  1525. *
  1526. * Caller must hold an exclusive lock on pgss->lock.
  1527. */
  1528. static void
  1529. entry_dealloc(void)
  1530. {
  1531. HASH_SEQ_STATUS hash_seq;
  1532. pgssEntry **entries;
  1533. pgssEntry *entry;
  1534. int nvictims;
  1535. int i;
  1536. Size tottextlen;
  1537. int nvalidtexts;
  1538. /*
  1539. * Sort entries by usage and deallocate USAGE_DEALLOC_PERCENT of them.
  1540. * While we're scanning the table, apply the decay factor to the usage
  1541. * values, and update the mean query length.
  1542. *
  1543. * Note that the mean query length is almost immediately obsolete, since
  1544. * we compute it before not after discarding the least-used entries.
  1545. * Hopefully, that doesn't affect the mean too much; it doesn't seem worth
  1546. * making two passes to get a more current result. Likewise, the new
  1547. * cur_median_usage includes the entries we're about to zap.
  1548. */
  1549. entries = palloc(hash_get_num_entries(pgss_hash) * sizeof(pgssEntry *));
  1550. i = 0;
  1551. tottextlen = 0;
  1552. nvalidtexts = 0;
  1553. hash_seq_init(&hash_seq, pgss_hash);
  1554. while ((entry = hash_seq_search(&hash_seq)) != NULL)
  1555. {
  1556. entries[i++] = entry;
  1557. /* "Sticky" entries get a different usage decay rate. */
  1558. if (entry->counters.calls == 0)
  1559. entry->counters.usage *= STICKY_DECREASE_FACTOR;
  1560. else
  1561. entry->counters.usage *= USAGE_DECREASE_FACTOR;
  1562. /* In the mean length computation, ignore dropped texts. */
  1563. if (entry->query_len >= 0)
  1564. {
  1565. tottextlen += entry->query_len + 1;
  1566. nvalidtexts++;
  1567. }
  1568. }
  1569. /* Sort into increasing order by usage */
  1570. qsort(entries, i, sizeof(pgssEntry *), entry_cmp);
  1571. /* Record the (approximate) median usage */
  1572. if (i > 0)
  1573. pgss->cur_median_usage = entries[i / 2]->counters.usage;
  1574. /* Record the mean query length */
  1575. if (nvalidtexts > 0)
  1576. pgss->mean_query_len = tottextlen / nvalidtexts;
  1577. else
  1578. pgss->mean_query_len = ASSUMED_LENGTH_INIT;
  1579. /* Now zap an appropriate fraction of lowest-usage entries */
  1580. nvictims = Max(10, i * USAGE_DEALLOC_PERCENT / 100);
  1581. nvictims = Min(nvictims, i);
  1582. for (i = 0; i < nvictims; i++)
  1583. {
  1584. hash_search(pgss_hash, &entries[i]->key, HASH_REMOVE, NULL);
  1585. }
  1586. pfree(entries);
  1587. }
  1588. /*
  1589. * Given a query string (not necessarily null-terminated), allocate a new
  1590. * entry in the external query text file and store the string there.
  1591. *
  1592. * If successful, returns true, and stores the new entry's offset in the file
  1593. * into *query_offset. Also, if gc_count isn't NULL, *gc_count is set to the
  1594. * number of garbage collections that have occurred so far.
  1595. *
  1596. * On failure, returns false.
  1597. *
  1598. * At least a shared lock on pgss->lock must be held by the caller, so as
  1599. * to prevent a concurrent garbage collection. Share-lock-holding callers
  1600. * should pass a gc_count pointer to obtain the number of garbage collections,
  1601. * so that they can recheck the count after obtaining exclusive lock to
  1602. * detect whether a garbage collection occurred (and removed this entry).
  1603. */
  1604. static bool
  1605. qtext_store(const char *query, int query_len,
  1606. Size *query_offset, int *gc_count)
  1607. {
  1608. Size off;
  1609. int fd;
  1610. /*
  1611. * We use a spinlock to protect extent/n_writers/gc_count, so that
  1612. * multiple processes may execute this function concurrently.
  1613. */
  1614. {
  1615. volatile pgssSharedState *s = (volatile pgssSharedState *) pgss;
  1616. SpinLockAcquire(&s->mutex);
  1617. off = s->extent;
  1618. s->extent += query_len + 1;
  1619. s->n_writers++;
  1620. if (gc_count)
  1621. *gc_count = s->gc_count;
  1622. SpinLockRelease(&s->mutex);
  1623. }
  1624. *query_offset = off;
  1625. /* Now write the data into the successfully-reserved part of the file */
  1626. fd = OpenTransientFile(PGSS_TEXT_FILE, O_RDWR | O_CREAT | PG_BINARY);
  1627. if (fd < 0)
  1628. goto error;
  1629. if (lseek(fd, off, SEEK_SET) != off)
  1630. goto error;
  1631. if (write(fd, query, query_len) != query_len)
  1632. goto error;
  1633. if (write(fd, "\0", 1) != 1)
  1634. goto error;
  1635. CloseTransientFile(fd);
  1636. /* Mark our write complete */
  1637. {
  1638. volatile pgssSharedState *s = (volatile pgssSharedState *) pgss;
  1639. SpinLockAcquire(&s->mutex);
  1640. s->n_writers--;
  1641. SpinLockRelease(&s->mutex);
  1642. }
  1643. return true;
  1644. error:
  1645. ereport(LOG,
  1646. (errcode_for_file_access(),
  1647. errmsg("could not write file \"%s\": %m",
  1648. PGSS_TEXT_FILE)));
  1649. if (fd >= 0)
  1650. CloseTransientFile(fd);
  1651. /* Mark our write complete */
  1652. {
  1653. volatile pgssSharedState *s = (volatile pgssSharedState *) pgss;
  1654. SpinLockAcquire(&s->mutex);
  1655. s->n_writers--;
  1656. SpinLockRelease(&s->mutex);
  1657. }
  1658. return false;
  1659. }
  1660. /*
  1661. * Read the external query text file into a malloc'd buffer.
  1662. *
  1663. * Returns NULL (without throwing an error) if unable to read, eg
  1664. * file not there or insufficient memory.
  1665. *
  1666. * On success, the buffer size is also returned into *buffer_size.
  1667. *
  1668. * This can be called without any lock on pgss->lock, but in that case
  1669. * the caller is responsible for verifying that the result is sane.
  1670. */
  1671. static char *
  1672. qtext_load_file(Size *buffer_size)
  1673. {
  1674. char *buf;
  1675. int fd;
  1676. struct stat stat;
  1677. fd = OpenTransientFile(PGSS_TEXT_FILE, O_RDONLY | PG_BINARY);
  1678. if (fd < 0)
  1679. {
  1680. if (errno != ENOENT)
  1681. ereport(LOG,
  1682. (errcode_for_file_access(),
  1683. errmsg("could not read file \"%s\": %m",
  1684. PGSS_TEXT_FILE)));
  1685. return NULL;
  1686. }
  1687. /* Get file length */
  1688. if (fstat(fd, &stat))
  1689. {
  1690. ereport(LOG,
  1691. (errcode_for_file_access(),
  1692. errmsg("could not stat file \"%s\": %m",
  1693. PGSS_TEXT_FILE)));
  1694. CloseTransientFile(fd);
  1695. return NULL;
  1696. }
  1697. /* Allocate buffer; beware that off_t might be wider than size_t */
  1698. if (stat.st_size <= MaxAllocHugeSize)
  1699. buf = (char *) malloc(stat.st_size);
  1700. else
  1701. buf = NULL;
  1702. if (buf == NULL)
  1703. {
  1704. ereport(LOG,
  1705. (errcode(ERRCODE_OUT_OF_MEMORY),
  1706. errmsg("out of memory"),
  1707. errdetail("Could not allocate enough memory to read file \"%s\".",
  1708. PGSS_TEXT_FILE)));
  1709. CloseTransientFile(fd);
  1710. return NULL;
  1711. }
  1712. /*
  1713. * OK, slurp in the file. If we get a short read and errno doesn't get
  1714. * set, the reason is probably that garbage collection truncated the file
  1715. * since we did the fstat(), so we don't log a complaint --- but we don't
  1716. * return the data, either, since it's most likely corrupt due to
  1717. * concurrent writes from garbage collection.
  1718. */
  1719. errno = 0;
  1720. if (read(fd, buf, stat.st_size) != stat.st_size)
  1721. {
  1722. if (errno)
  1723. ereport(LOG,
  1724. (errcode_for_file_access(),
  1725. errmsg("could not read file \"%s\": %m",
  1726. PGSS_TEXT_FILE)));
  1727. free(buf);
  1728. CloseTransientFile(fd);
  1729. return NULL;
  1730. }
  1731. CloseTransientFile(fd);
  1732. *buffer_size = stat.st_size;
  1733. return buf;
  1734. }
  1735. /*
  1736. * Locate a query text in the file image previously read by qtext_load_file().
  1737. *
  1738. * We validate the given offset/length, and return NULL if bogus. Otherwise,
  1739. * the result points to a null-terminated string within the buffer.
  1740. */
  1741. static char *
  1742. qtext_fetch(Size query_offset, int query_len,
  1743. char *buffer, Size buffer_size)
  1744. {
  1745. /* File read failed? */
  1746. if (buffer == NULL)
  1747. return NULL;
  1748. /* Bogus offset/length? */
  1749. if (query_len < 0 ||
  1750. query_offset + query_len >= buffer_size)
  1751. return NULL;
  1752. /* As a further sanity check, make sure there's a trailing null */
  1753. if (buffer[query_offset + query_len] != '\0')
  1754. return NULL;
  1755. /* Looks OK */
  1756. return buffer + query_offset;
  1757. }
  1758. /*
  1759. * Do we need to garbage-collect the external query text file?
  1760. *
  1761. * Caller should hold at least a shared lock on pgss->lock.
  1762. */
  1763. static bool
  1764. need_gc_qtexts(void)
  1765. {
  1766. Size extent;
  1767. /* Read shared extent pointer */
  1768. {
  1769. volatile pgssSharedState *s = (volatile pgssSharedState *) pgss;
  1770. SpinLockAcquire(&s->mutex);
  1771. extent = s->extent;
  1772. SpinLockRelease(&s->mutex);
  1773. }
  1774. /* Don't proceed if file does not exceed 512 bytes per possible entry */
  1775. if (extent < 512 * pgss_max)
  1776. return false;
  1777. /*
  1778. * Don't proceed if file is less than about 50% bloat. Nothing can or
  1779. * should be done in the event of unusually large query texts accounting
  1780. * for file's large size. We go to the trouble of maintaining the mean
  1781. * query length in order to prevent garbage collection from thrashing
  1782. * uselessly.
  1783. */
  1784. if (extent < pgss->mean_query_len * pgss_max * 2)
  1785. return false;
  1786. return true;
  1787. }
  1788. /*
  1789. * Garbage-collect orphaned query texts in external file.
  1790. *
  1791. * This won't be called often in the typical case, since it's likely that
  1792. * there won't be too much churn, and besides, a similar compaction process
  1793. * occurs when serializing to disk at shutdown or as part of resetting.
  1794. * Despite this, it seems prudent to plan for the edge case where the file
  1795. * becomes unreasonably large, with no other method of compaction likely to
  1796. * occur in the foreseeable future.
  1797. *
  1798. * The caller must hold an exclusive lock on pgss->lock.
  1799. *
  1800. * At the first sign of trouble we unlink the query text file to get a clean
  1801. * slate (although existing statistics are retained), rather than risk
  1802. * thrashing by allowing the same problem case to recur indefinitely.
  1803. */
  1804. static void
  1805. gc_qtexts(void)
  1806. {
  1807. char *qbuffer;
  1808. Size qbuffer_size;
  1809. FILE *qfile = NULL;
  1810. HASH_SEQ_STATUS hash_seq;
  1811. pgssEntry *entry;
  1812. Size extent;
  1813. int nentries;
  1814. /*
  1815. * When called from pgss_store, some other session might have proceeded
  1816. * with garbage collection in the no-lock-held interim of lock strength
  1817. * escalation. Check once more that this is actually necessary.
  1818. */
  1819. if (!need_gc_qtexts())
  1820. return;
  1821. /*
  1822. * Load the old texts file. If we fail (out of memory, for instance),
  1823. * invalidate query texts. Hopefully this is rare. It might seem better
  1824. * to leave things alone on an OOM failure, but the problem is that the
  1825. * file is only going to get bigger; hoping for a future non-OOM result is
  1826. * risky and can easily lead to complete denial of service.
  1827. */
  1828. qbuffer = qtext_load_file(&qbuffer_size);
  1829. if (qbuffer == NULL)
  1830. goto gc_fail;
  1831. /*
  1832. * We overwrite the query texts file in place, so as to reduce the risk of
  1833. * an out-of-disk-space failure. Since the file is guaranteed not to get
  1834. * larger, this should always work on traditional filesystems; though we
  1835. * could still lose on copy-on-write filesystems.
  1836. */
  1837. qfile = AllocateFile(PGSS_TEXT_FILE, PG_BINARY_W);
  1838. if (qfile == NULL)
  1839. {
  1840. ereport(LOG,
  1841. (errcode_for_file_access(),
  1842. errmsg("could not write file \"%s\": %m",
  1843. PGSS_TEXT_FILE)));
  1844. goto gc_fail;
  1845. }
  1846. extent = 0;
  1847. nentries = 0;
  1848. hash_seq_init(&hash_seq, pgss_hash);
  1849. while ((entry = hash_seq_search(&hash_seq)) != NULL)
  1850. {
  1851. int query_len = entry->query_len;
  1852. char *qry = qtext_fetch(entry->query_offset,
  1853. query_len,
  1854. qbuffer,
  1855. qbuffer_size);
  1856. if (qry == NULL)
  1857. {
  1858. /* Trouble ... drop the text */
  1859. entry->query_offset = 0;
  1860. entry->query_len = -1;
  1861. /* entry will not be counted in mean query length computation */
  1862. continue;
  1863. }
  1864. if (fwrite(qry, 1, query_len + 1, qfile) != query_len + 1)
  1865. {
  1866. ereport(LOG,
  1867. (errcode_for_file_access(),
  1868. errmsg("could not write file \"%s\": %m",
  1869. PGSS_TEXT_FILE)));
  1870. hash_seq_term(&hash_seq);
  1871. goto gc_fail;
  1872. }
  1873. entry->query_offset = extent;
  1874. extent += query_len + 1;
  1875. nentries++;
  1876. }
  1877. /*
  1878. * Truncate away any now-unused space. If this fails for some odd reason,
  1879. * we log it, but there's no need to fail.
  1880. */
  1881. if (ftruncate(fileno(qfile), extent) != 0)
  1882. ereport(LOG,
  1883. (errcode_for_file_access(),
  1884. errmsg("could not truncate file \"%s\": %m",
  1885. PGSS_TEXT_FILE)));
  1886. if (FreeFile(qfile))
  1887. {
  1888. ereport(LOG,
  1889. (errcode_for_file_access(),
  1890. errmsg("could not write file \"%s\": %m",
  1891. PGSS_TEXT_FILE)));
  1892. qfile = NULL;
  1893. goto gc_fail;
  1894. }
  1895. elog(DEBUG1, "pgss gc of queries file shrunk size from %zu to %zu",
  1896. pgss->extent, extent);
  1897. /* Reset the shared extent pointer */
  1898. pgss->extent = extent;
  1899. /*
  1900. * Also update the mean query length, to be sure that need_gc_qtexts()
  1901. * won't still think we have a problem.
  1902. */
  1903. if (nentries > 0)
  1904. pgss->mean_query_len = extent / nentries;
  1905. else
  1906. pgss->mean_query_len = ASSUMED_LENGTH_INIT;
  1907. free(qbuffer);
  1908. /*
  1909. * OK, count a garbage collection cycle. (Note: even though we have
  1910. * exclusive lock on pgss->lock, we must take pgss->mutex for this, since
  1911. * other processes may examine gc_count while holding only the mutex.
  1912. * Also, we have to advance the count *after* we've rewritten the file,
  1913. * else other processes might not realize they read a stale file.)
  1914. */
  1915. record_gc_qtexts();
  1916. return;
  1917. gc_fail:
  1918. /* clean up resources */
  1919. if (qfile)
  1920. FreeFile(qfile);
  1921. if (qbuffer)
  1922. free(qbuffer);
  1923. /*
  1924. * Since the contents of the external file are now uncertain, mark all
  1925. * hashtable entries as having invalid texts.
  1926. */
  1927. hash_seq_init(&hash_seq, pgss_hash);
  1928. while ((entry = hash_seq_search(&hash_seq)) != NULL)
  1929. {
  1930. entry->query_offset = 0;
  1931. entry->query_len = -1;
  1932. }
  1933. /*
  1934. * Destroy the query text file and create a new, empty one
  1935. */
  1936. (void) unlink(PGSS_TEXT_FILE);
  1937. qfile = AllocateFile(PGSS_TEXT_FILE, PG_BINARY_W);
  1938. if (qfile == NULL)
  1939. ereport(LOG,
  1940. (errcode_for_file_access(),
  1941. errmsg("could not recreate file \"%s\": %m",
  1942. PGSS_TEXT_FILE)));
  1943. else
  1944. FreeFile(qfile);
  1945. /* Reset the shared extent pointer */
  1946. pgss->extent = 0;
  1947. /* Reset mean_query_len to match the new state */
  1948. pgss->mean_query_len = ASSUMED_LENGTH_INIT;
  1949. /*
  1950. * Bump the GC count even though we failed.
  1951. *
  1952. * This is needed to make concurrent readers of file without any lock on
  1953. * pgss->lock notice existence of new version of file. Once readers
  1954. * subsequently observe a change in GC count with pgss->lock held, that
  1955. * forces a safe reopen of file. Writers also require that we bump here,
  1956. * of course. (As required by locking protocol, readers and writers don't
  1957. * trust earlier file contents until gc_count is found unchanged after
  1958. * pgss->lock acquired in shared or exclusive mode respectively.)
  1959. */
  1960. record_gc_qtexts();
  1961. }
  1962. /*
  1963. * Release all entries.
  1964. */
  1965. static void
  1966. entry_reset(void)
  1967. {
  1968. HASH_SEQ_STATUS hash_seq;
  1969. pgssEntry *entry;
  1970. FILE *qfile;
  1971. LWLockAcquire(pgss->lock, LW_EXCLUSIVE);
  1972. hash_seq_init(&hash_seq, pgss_hash);
  1973. while ((entry = hash_seq_search(&hash_seq)) != NULL)
  1974. {
  1975. hash_search(pgss_hash, &entry->key, HASH_REMOVE, NULL);
  1976. }
  1977. /*
  1978. * Write new empty query file, perhaps even creating a new one to recover
  1979. * if the file was missing.
  1980. */
  1981. qfile = AllocateFile(PGSS_TEXT_FILE, PG_BINARY_W);
  1982. if (qfile == NULL)
  1983. {
  1984. ereport(LOG,
  1985. (errcode_for_file_access(),
  1986. errmsg("could not create file \"%s\": %m",
  1987. PGSS_TEXT_FILE)));
  1988. goto done;
  1989. }
  1990. /* If ftruncate fails, log it, but it's not a fatal problem */
  1991. if (ftruncate(fileno(qfile), 0) != 0)
  1992. ereport(LOG,
  1993. (errcode_for_file_access(),
  1994. errmsg("could not truncate file \"%s\": %m",
  1995. PGSS_TEXT_FILE)));
  1996. FreeFile(qfile);
  1997. done:
  1998. pgss->extent = 0;
  1999. /* This counts as a query text garbage collection for our purposes */
  2000. record_gc_qtexts();
  2001. LWLockRelease(pgss->lock);
  2002. }
  2003. /*
  2004. * AppendJumble: Append a value that is substantive in a given query to
  2005. * the current jumble.
  2006. */
  2007. static void
  2008. AppendJumble(pgssJumbleState *jstate, const unsigned char *item, Size size)
  2009. {
  2010. unsigned char *jumble = jstate->jumble;
  2011. Size jumble_len = jstate->jumble_len;
  2012. /*
  2013. * Whenever the jumble buffer is full, we hash the current contents and
  2014. * reset the buffer to contain just that hash value, thus relying on the
  2015. * hash to summarize everything so far.
  2016. */
  2017. while (size > 0)
  2018. {
  2019. Size part_size;
  2020. if (jumble_len >= JUMBLE_SIZE)
  2021. {
  2022. uint64 start_hash;
  2023. start_hash = DatumGetUInt64(hash_any_extended(jumble,
  2024. JUMBLE_SIZE, 0));
  2025. memcpy(jumble, &start_hash, sizeof(start_hash));
  2026. jumble_len = sizeof(start_hash);
  2027. }
  2028. part_size = Min(size, JUMBLE_SIZE - jumble_len);
  2029. memcpy(jumble + jumble_len, item, part_size);
  2030. jumble_len += part_size;
  2031. item += part_size;
  2032. size -= part_size;
  2033. }
  2034. jstate->jumble_len = jumble_len;
  2035. }
  2036. /*
  2037. * Wrappers around AppendJumble to encapsulate details of serialization
  2038. * of individual local variable elements.
  2039. */
  2040. #define APP_JUMB(item) \
  2041. AppendJumble(jstate, (const unsigned char *) &(item), sizeof(item))
  2042. #define APP_JUMB_STRING(str) \
  2043. AppendJumble(jstate, (const unsigned char *) (str), strlen(str) + 1)
  2044. /*
  2045. * JumbleQuery: Selectively serialize the query tree, appending significant
  2046. * data to the "query jumble" while ignoring nonsignificant data.
  2047. *
  2048. * Rule of thumb for what to include is that we should ignore anything not
  2049. * semantically significant (such as alias names) as well as anything that can
  2050. * be deduced from child nodes (else we'd just be double-hashing that piece
  2051. * of information).
  2052. */
  2053. static void
  2054. JumbleQuery(pgssJumbleState *jstate, Query *query)
  2055. {
  2056. Assert(IsA(query, Query));
  2057. Assert(query->utilityStmt == NULL);
  2058. APP_JUMB(query->commandType);
  2059. /* resultRelation is usually predictable from commandType */
  2060. JumbleExpr(jstate, (Node *) query->cteList);
  2061. JumbleRangeTable(jstate, query->rtable);
  2062. JumbleExpr(jstate, (Node *) query->jointree);
  2063. JumbleExpr(jstate, (Node *) query->targetList);
  2064. JumbleExpr(jstate, (Node *) query->onConflict);
  2065. JumbleExpr(jstate, (Node *) query->returningList);
  2066. JumbleExpr(jstate, (Node *) query->groupClause);
  2067. JumbleExpr(jstate, (Node *) query->groupingSets);
  2068. JumbleExpr(jstate, query->havingQual);
  2069. JumbleExpr(jstate, (Node *) query->windowClause);
  2070. JumbleExpr(jstate, (Node *) query->distinctClause);
  2071. JumbleExpr(jstate, (Node *) query->sortClause);
  2072. JumbleExpr(jstate, query->limitOffset);
  2073. JumbleExpr(jstate, query->limitCount);
  2074. /* we ignore rowMarks */
  2075. JumbleExpr(jstate, query->setOperations);
  2076. }
  2077. /*
  2078. * Jumble a range table
  2079. */
  2080. static void
  2081. JumbleRangeTable(pgssJumbleState *jstate, List *rtable)
  2082. {
  2083. ListCell *lc;
  2084. foreach(lc, rtable)
  2085. {
  2086. RangeTblEntry *rte = lfirst_node(RangeTblEntry, lc);
  2087. APP_JUMB(rte->rtekind);
  2088. switch (rte->rtekind)
  2089. {
  2090. case RTE_RELATION:
  2091. APP_JUMB(rte->relid);
  2092. JumbleExpr(jstate, (Node *) rte->tablesample);
  2093. break;
  2094. case RTE_SUBQUERY:
  2095. JumbleQuery(jstate, rte->subquery);
  2096. break;
  2097. case RTE_JOIN:
  2098. APP_JUMB(rte->jointype);
  2099. break;
  2100. case RTE_FUNCTION:
  2101. JumbleExpr(jstate, (Node *) rte->functions);
  2102. break;
  2103. case RTE_TABLEFUNC:
  2104. JumbleExpr(jstate, (Node *) rte->tablefunc);
  2105. break;
  2106. case RTE_VALUES:
  2107. JumbleExpr(jstate, (Node *) rte->values_lists);
  2108. break;
  2109. case RTE_CTE:
  2110. /*
  2111. * Depending on the CTE name here isn't ideal, but it's the
  2112. * only info we have to identify the referenced WITH item.
  2113. */
  2114. APP_JUMB_STRING(rte->ctename);
  2115. APP_JUMB(rte->ctelevelsup);
  2116. break;
  2117. case RTE_NAMEDTUPLESTORE:
  2118. APP_JUMB_STRING(rte->enrname);
  2119. break;
  2120. default:
  2121. elog(ERROR, "unrecognized RTE kind: %d", (int) rte->rtekind);
  2122. break;
  2123. }
  2124. }
  2125. }
  2126. /*
  2127. * Jumble an expression tree
  2128. *
  2129. * In general this function should handle all the same node types that
  2130. * expression_tree_walker() does, and therefore it's coded to be as parallel
  2131. * to that function as possible. However, since we are only invoked on
  2132. * queries immediately post-parse-analysis, we need not handle node types
  2133. * that only appear in planning.
  2134. *
  2135. * Note: the reason we don't simply use expression_tree_walker() is that the
  2136. * point of that function is to support tree walkers that don't care about
  2137. * most tree node types, but here we care about all types. We should complain
  2138. * about any unrecognized node type.
  2139. */
  2140. static void
  2141. JumbleExpr(pgssJumbleState *jstate, Node *node)
  2142. {
  2143. ListCell *temp;
  2144. if (node == NULL)
  2145. return;
  2146. /* Guard against stack overflow due to overly complex expressions */
  2147. check_stack_depth();
  2148. /*
  2149. * We always emit the node's NodeTag, then any additional fields that are
  2150. * considered significant, and then we recurse to any child nodes.
  2151. */
  2152. APP_JUMB(node->type);
  2153. switch (nodeTag(node))
  2154. {
  2155. case T_Var:
  2156. {
  2157. Var *var = (Var *) node;
  2158. APP_JUMB(var->varno);
  2159. APP_JUMB(var->varattno);
  2160. APP_JUMB(var->varlevelsup);
  2161. }
  2162. break;
  2163. case T_Const:
  2164. {
  2165. Const *c = (Const *) node;
  2166. /* We jumble only the constant's type, not its value */
  2167. APP_JUMB(c->consttype);
  2168. /* Also, record its parse location for query normalization */
  2169. RecordConstLocation(jstate, c->location);
  2170. }
  2171. break;
  2172. case T_Param:
  2173. {
  2174. Param *p = (Param *) node;
  2175. APP_JUMB(p->paramkind);
  2176. APP_JUMB(p->paramid);
  2177. APP_JUMB(p->paramtype);
  2178. /* Also, track the highest external Param id */
  2179. if (p->paramkind == PARAM_EXTERN &&
  2180. p->paramid > jstate->highest_extern_param_id)
  2181. jstate->highest_extern_param_id = p->paramid;
  2182. }
  2183. break;
  2184. case T_Aggref:
  2185. {
  2186. Aggref *expr = (Aggref *) node;
  2187. APP_JUMB(expr->aggfnoid);
  2188. JumbleExpr(jstate, (Node *) expr->aggdirectargs);
  2189. JumbleExpr(jstate, (Node *) expr->args);
  2190. JumbleExpr(jstate, (Node *) expr->aggorder);
  2191. JumbleExpr(jstate, (Node *) expr->aggdistinct);
  2192. JumbleExpr(jstate, (Node *) expr->aggfilter);
  2193. }
  2194. break;
  2195. case T_GroupingFunc:
  2196. {
  2197. GroupingFunc *grpnode = (GroupingFunc *) node;
  2198. JumbleExpr(jstate, (Node *) grpnode->refs);
  2199. }
  2200. break;
  2201. case T_WindowFunc:
  2202. {
  2203. WindowFunc *expr = (WindowFunc *) node;
  2204. APP_JUMB(expr->winfnoid);
  2205. APP_JUMB(expr->winref);
  2206. JumbleExpr(jstate, (Node *) expr->args);
  2207. JumbleExpr(jstate, (Node *) expr->aggfilter);
  2208. }
  2209. break;
  2210. case T_ArrayRef:
  2211. {
  2212. ArrayRef *aref = (ArrayRef *) node;
  2213. JumbleExpr(jstate, (Node *) aref->refupperindexpr);
  2214. JumbleExpr(jstate, (Node *) aref->reflowerindexpr);
  2215. JumbleExpr(jstate, (Node *) aref->refexpr);
  2216. JumbleExpr(jstate, (Node *) aref->refassgnexpr);
  2217. }
  2218. break;
  2219. case T_FuncExpr:
  2220. {
  2221. FuncExpr *expr = (FuncExpr *) node;
  2222. APP_JUMB(expr->funcid);
  2223. JumbleExpr(jstate, (Node *) expr->args);
  2224. }
  2225. break;
  2226. case T_NamedArgExpr:
  2227. {
  2228. NamedArgExpr *nae = (NamedArgExpr *) node;
  2229. APP_JUMB(nae->argnumber);
  2230. JumbleExpr(jstate, (Node *) nae->arg);
  2231. }
  2232. break;
  2233. case T_OpExpr:
  2234. case T_DistinctExpr: /* struct-equivalent to OpExpr */
  2235. case T_NullIfExpr: /* struct-equivalent to OpExpr */
  2236. {
  2237. OpExpr *expr = (OpExpr *) node;
  2238. APP_JUMB(expr->opno);
  2239. JumbleExpr(jstate, (Node *) expr->args);
  2240. }
  2241. break;
  2242. case T_ScalarArrayOpExpr:
  2243. {
  2244. ScalarArrayOpExpr *expr = (ScalarArrayOpExpr *) node;
  2245. APP_JUMB(expr->opno);
  2246. APP_JUMB(expr->useOr);
  2247. JumbleExpr(jstate, (Node *) expr->args);
  2248. }
  2249. break;
  2250. case T_BoolExpr:
  2251. {
  2252. BoolExpr *expr = (BoolExpr *) node;
  2253. APP_JUMB(expr->boolop);
  2254. JumbleExpr(jstate, (Node *) expr->args);
  2255. }
  2256. break;
  2257. case T_SubLink:
  2258. {
  2259. SubLink *sublink = (SubLink *) node;
  2260. APP_JUMB(sublink->subLinkType);
  2261. APP_JUMB(sublink->subLinkId);
  2262. JumbleExpr(jstate, (Node *) sublink->testexpr);
  2263. JumbleQuery(jstate, castNode(Query, sublink->subselect));
  2264. }
  2265. break;
  2266. case T_FieldSelect:
  2267. {
  2268. FieldSelect *fs = (FieldSelect *) node;
  2269. APP_JUMB(fs->fieldnum);
  2270. JumbleExpr(jstate, (Node *) fs->arg);
  2271. }
  2272. break;
  2273. case T_FieldStore:
  2274. {
  2275. FieldStore *fstore = (FieldStore *) node;
  2276. JumbleExpr(jstate, (Node *) fstore->arg);
  2277. JumbleExpr(jstate, (Node *) fstore->newvals);
  2278. }
  2279. break;
  2280. case T_RelabelType:
  2281. {
  2282. RelabelType *rt = (RelabelType *) node;
  2283. APP_JUMB(rt->resulttype);
  2284. JumbleExpr(jstate, (Node *) rt->arg);
  2285. }
  2286. break;
  2287. case T_CoerceViaIO:
  2288. {
  2289. CoerceViaIO *cio = (CoerceViaIO *) node;
  2290. APP_JUMB(cio->resulttype);
  2291. JumbleExpr(jstate, (Node *) cio->arg);
  2292. }
  2293. break;
  2294. case T_ArrayCoerceExpr:
  2295. {
  2296. ArrayCoerceExpr *acexpr = (ArrayCoerceExpr *) node;
  2297. APP_JUMB(acexpr->resulttype);
  2298. JumbleExpr(jstate, (Node *) acexpr->arg);
  2299. JumbleExpr(jstate, (Node *) acexpr->elemexpr);
  2300. }
  2301. break;
  2302. case T_ConvertRowtypeExpr:
  2303. {
  2304. ConvertRowtypeExpr *crexpr = (ConvertRowtypeExpr *) node;
  2305. APP_JUMB(crexpr->resulttype);
  2306. JumbleExpr(jstate, (Node *) crexpr->arg);
  2307. }
  2308. break;
  2309. case T_CollateExpr:
  2310. {
  2311. CollateExpr *ce = (CollateExpr *) node;
  2312. APP_JUMB(ce->collOid);
  2313. JumbleExpr(jstate, (Node *) ce->arg);
  2314. }
  2315. break;
  2316. case T_CaseExpr:
  2317. {
  2318. CaseExpr *caseexpr = (CaseExpr *) node;
  2319. JumbleExpr(jstate, (Node *) caseexpr->arg);
  2320. foreach(temp, caseexpr->args)
  2321. {
  2322. CaseWhen *when = lfirst_node(CaseWhen, temp);
  2323. JumbleExpr(jstate, (Node *) when->expr);
  2324. JumbleExpr(jstate, (Node *) when->result);
  2325. }
  2326. JumbleExpr(jstate, (Node *) caseexpr->defresult);
  2327. }
  2328. break;
  2329. case T_CaseTestExpr:
  2330. {
  2331. CaseTestExpr *ct = (CaseTestExpr *) node;
  2332. APP_JUMB(ct->typeId);
  2333. }
  2334. break;
  2335. case T_ArrayExpr:
  2336. JumbleExpr(jstate, (Node *) ((ArrayExpr *) node)->elements);
  2337. break;
  2338. case T_RowExpr:
  2339. JumbleExpr(jstate, (Node *) ((RowExpr *) node)->args);
  2340. break;
  2341. case T_RowCompareExpr:
  2342. {
  2343. RowCompareExpr *rcexpr = (RowCompareExpr *) node;
  2344. APP_JUMB(rcexpr->rctype);
  2345. JumbleExpr(jstate, (Node *) rcexpr->largs);
  2346. JumbleExpr(jstate, (Node *) rcexpr->rargs);
  2347. }
  2348. break;
  2349. case T_CoalesceExpr:
  2350. JumbleExpr(jstate, (Node *) ((CoalesceExpr *) node)->args);
  2351. break;
  2352. case T_MinMaxExpr:
  2353. {
  2354. MinMaxExpr *mmexpr = (MinMaxExpr *) node;
  2355. APP_JUMB(mmexpr->op);
  2356. JumbleExpr(jstate, (Node *) mmexpr->args);
  2357. }
  2358. break;
  2359. case T_SQLValueFunction:
  2360. {
  2361. SQLValueFunction *svf = (SQLValueFunction *) node;
  2362. APP_JUMB(svf->op);
  2363. /* type is fully determined by op */
  2364. APP_JUMB(svf->typmod);
  2365. }
  2366. break;
  2367. case T_XmlExpr:
  2368. {
  2369. XmlExpr *xexpr = (XmlExpr *) node;
  2370. APP_JUMB(xexpr->op);
  2371. JumbleExpr(jstate, (Node *) xexpr->named_args);
  2372. JumbleExpr(jstate, (Node *) xexpr->args);
  2373. }
  2374. break;
  2375. case T_NullTest:
  2376. {
  2377. NullTest *nt = (NullTest *) node;
  2378. APP_JUMB(nt->nulltesttype);
  2379. JumbleExpr(jstate, (Node *) nt->arg);
  2380. }
  2381. break;
  2382. case T_BooleanTest:
  2383. {
  2384. BooleanTest *bt = (BooleanTest *) node;
  2385. APP_JUMB(bt->booltesttype);
  2386. JumbleExpr(jstate, (Node *) bt->arg);
  2387. }
  2388. break;
  2389. case T_CoerceToDomain:
  2390. {
  2391. CoerceToDomain *cd = (CoerceToDomain *) node;
  2392. APP_JUMB(cd->resulttype);
  2393. JumbleExpr(jstate, (Node *) cd->arg);
  2394. }
  2395. break;
  2396. case T_CoerceToDomainValue:
  2397. {
  2398. CoerceToDomainValue *cdv = (CoerceToDomainValue *) node;
  2399. APP_JUMB(cdv->typeId);
  2400. }
  2401. break;
  2402. case T_SetToDefault:
  2403. {
  2404. SetToDefault *sd = (SetToDefault *) node;
  2405. APP_JUMB(sd->typeId);
  2406. }
  2407. break;
  2408. case T_CurrentOfExpr:
  2409. {
  2410. CurrentOfExpr *ce = (CurrentOfExpr *) node;
  2411. APP_JUMB(ce->cvarno);
  2412. if (ce->cursor_name)
  2413. APP_JUMB_STRING(ce->cursor_name);
  2414. APP_JUMB(ce->cursor_param);
  2415. }
  2416. break;
  2417. case T_NextValueExpr:
  2418. {
  2419. NextValueExpr *nve = (NextValueExpr *) node;
  2420. APP_JUMB(nve->seqid);
  2421. APP_JUMB(nve->typeId);
  2422. }
  2423. break;
  2424. case T_InferenceElem:
  2425. {
  2426. InferenceElem *ie = (InferenceElem *) node;
  2427. APP_JUMB(ie->infercollid);
  2428. APP_JUMB(ie->inferopclass);
  2429. JumbleExpr(jstate, ie->expr);
  2430. }
  2431. break;
  2432. case T_TargetEntry:
  2433. {
  2434. TargetEntry *tle = (TargetEntry *) node;
  2435. APP_JUMB(tle->resno);
  2436. APP_JUMB(tle->ressortgroupref);
  2437. JumbleExpr(jstate, (Node *) tle->expr);
  2438. }
  2439. break;
  2440. case T_RangeTblRef:
  2441. {
  2442. RangeTblRef *rtr = (RangeTblRef *) node;
  2443. APP_JUMB(rtr->rtindex);
  2444. }
  2445. break;
  2446. case T_JoinExpr:
  2447. {
  2448. JoinExpr *join = (JoinExpr *) node;
  2449. APP_JUMB(join->jointype);
  2450. APP_JUMB(join->isNatural);
  2451. APP_JUMB(join->rtindex);
  2452. JumbleExpr(jstate, join->larg);
  2453. JumbleExpr(jstate, join->rarg);
  2454. JumbleExpr(jstate, join->quals);
  2455. }
  2456. break;
  2457. case T_FromExpr:
  2458. {
  2459. FromExpr *from = (FromExpr *) node;
  2460. JumbleExpr(jstate, (Node *) from->fromlist);
  2461. JumbleExpr(jstate, from->quals);
  2462. }
  2463. break;
  2464. case T_OnConflictExpr:
  2465. {
  2466. OnConflictExpr *conf = (OnConflictExpr *) node;
  2467. APP_JUMB(conf->action);
  2468. JumbleExpr(jstate, (Node *) conf->arbiterElems);
  2469. JumbleExpr(jstate, conf->arbiterWhere);
  2470. JumbleExpr(jstate, (Node *) conf->onConflictSet);
  2471. JumbleExpr(jstate, conf->onConflictWhere);
  2472. APP_JUMB(conf->constraint);
  2473. APP_JUMB(conf->exclRelIndex);
  2474. JumbleExpr(jstate, (Node *) conf->exclRelTlist);
  2475. }
  2476. break;
  2477. case T_List:
  2478. foreach(temp, (List *) node)
  2479. {
  2480. JumbleExpr(jstate, (Node *) lfirst(temp));
  2481. }
  2482. break;
  2483. case T_IntList:
  2484. foreach(temp, (List *) node)
  2485. {
  2486. APP_JUMB(lfirst_int(temp));
  2487. }
  2488. break;
  2489. case T_SortGroupClause:
  2490. {
  2491. SortGroupClause *sgc = (SortGroupClause *) node;
  2492. APP_JUMB(sgc->tleSortGroupRef);
  2493. APP_JUMB(sgc->eqop);
  2494. APP_JUMB(sgc->sortop);
  2495. APP_JUMB(sgc->nulls_first);
  2496. }
  2497. break;
  2498. case T_GroupingSet:
  2499. {
  2500. GroupingSet *gsnode = (GroupingSet *) node;
  2501. JumbleExpr(jstate, (Node *) gsnode->content);
  2502. }
  2503. break;
  2504. case T_WindowClause:
  2505. {
  2506. WindowClause *wc = (WindowClause *) node;
  2507. APP_JUMB(wc->winref);
  2508. APP_JUMB(wc->frameOptions);
  2509. JumbleExpr(jstate, (Node *) wc->partitionClause);
  2510. JumbleExpr(jstate, (Node *) wc->orderClause);
  2511. JumbleExpr(jstate, wc->startOffset);
  2512. JumbleExpr(jstate, wc->endOffset);
  2513. }
  2514. break;
  2515. case T_CommonTableExpr:
  2516. {
  2517. CommonTableExpr *cte = (CommonTableExpr *) node;
  2518. /* we store the string name because RTE_CTE RTEs need it */
  2519. APP_JUMB_STRING(cte->ctename);
  2520. JumbleQuery(jstate, castNode(Query, cte->ctequery));
  2521. }
  2522. break;
  2523. case T_SetOperationStmt:
  2524. {
  2525. SetOperationStmt *setop = (SetOperationStmt *) node;
  2526. APP_JUMB(setop->op);
  2527. APP_JUMB(setop->all);
  2528. JumbleExpr(jstate, setop->larg);
  2529. JumbleExpr(jstate, setop->rarg);
  2530. }
  2531. break;
  2532. case T_RangeTblFunction:
  2533. {
  2534. RangeTblFunction *rtfunc = (RangeTblFunction *) node;
  2535. JumbleExpr(jstate, rtfunc->funcexpr);
  2536. }
  2537. break;
  2538. case T_TableFunc:
  2539. {
  2540. TableFunc *tablefunc = (TableFunc *) node;
  2541. JumbleExpr(jstate, tablefunc->docexpr);
  2542. JumbleExpr(jstate, tablefunc->rowexpr);
  2543. JumbleExpr(jstate, (Node *) tablefunc->colexprs);
  2544. }
  2545. break;
  2546. case T_TableSampleClause:
  2547. {
  2548. TableSampleClause *tsc = (TableSampleClause *) node;
  2549. APP_JUMB(tsc->tsmhandler);
  2550. JumbleExpr(jstate, (Node *) tsc->args);
  2551. JumbleExpr(jstate, (Node *) tsc->repeatable);
  2552. }
  2553. break;
  2554. default:
  2555. /* Only a warning, since we can stumble along anyway */
  2556. elog(WARNING, "unrecognized node type: %d",
  2557. (int) nodeTag(node));
  2558. break;
  2559. }
  2560. }
  2561. /*
  2562. * Record location of constant within query string of query tree
  2563. * that is currently being walked.
  2564. */
  2565. static void
  2566. RecordConstLocation(pgssJumbleState *jstate, int location)
  2567. {
  2568. /* -1 indicates unknown or undefined location */
  2569. if (location >= 0)
  2570. {
  2571. /* enlarge array if needed */
  2572. if (jstate->clocations_count >= jstate->clocations_buf_size)
  2573. {
  2574. jstate->clocations_buf_size *= 2;
  2575. jstate->clocations = (pgssLocationLen *)
  2576. repalloc(jstate->clocations,
  2577. jstate->clocations_buf_size *
  2578. sizeof(pgssLocationLen));
  2579. }
  2580. jstate->clocations[jstate->clocations_count].location = location;
  2581. /* initialize lengths to -1 to simplify fill_in_constant_lengths */
  2582. jstate->clocations[jstate->clocations_count].length = -1;
  2583. jstate->clocations_count++;
  2584. }
  2585. }
  2586. /*
  2587. * Generate a normalized version of the query string that will be used to
  2588. * represent all similar queries.
  2589. *
  2590. * Note that the normalized representation may well vary depending on
  2591. * just which "equivalent" query is used to create the hashtable entry.
  2592. * We assume this is OK.
  2593. *
  2594. * If query_loc > 0, then "query" has been advanced by that much compared to
  2595. * the original string start, so we need to translate the provided locations
  2596. * to compensate. (This lets us avoid re-scanning statements before the one
  2597. * of interest, so it's worth doing.)
  2598. *
  2599. * *query_len_p contains the input string length, and is updated with
  2600. * the result string length on exit. The resulting string might be longer
  2601. * or shorter depending on what happens with replacement of constants.
  2602. *
  2603. * Returns a palloc'd string.
  2604. */
  2605. static char *
  2606. generate_normalized_query(pgssJumbleState *jstate, const char *query,
  2607. int query_loc, int *query_len_p, int encoding)
  2608. {
  2609. char *norm_query;
  2610. int query_len = *query_len_p;
  2611. int i,
  2612. norm_query_buflen, /* Space allowed for norm_query */
  2613. len_to_wrt, /* Length (in bytes) to write */
  2614. quer_loc = 0, /* Source query byte location */
  2615. n_quer_loc = 0, /* Normalized query byte location */
  2616. last_off = 0, /* Offset from start for previous tok */
  2617. last_tok_len = 0; /* Length (in bytes) of that tok */
  2618. /*
  2619. * Get constants' lengths (core system only gives us locations). Note
  2620. * this also ensures the items are sorted by location.
  2621. */
  2622. fill_in_constant_lengths(jstate, query, query_loc);
  2623. /*
  2624. * Allow for $n symbols to be longer than the constants they replace.
  2625. * Constants must take at least one byte in text form, while a $n symbol
  2626. * certainly isn't more than 11 bytes, even if n reaches INT_MAX. We
  2627. * could refine that limit based on the max value of n for the current
  2628. * query, but it hardly seems worth any extra effort to do so.
  2629. */
  2630. norm_query_buflen = query_len + jstate->clocations_count * 10;
  2631. /* Allocate result buffer */
  2632. norm_query = palloc(norm_query_buflen + 1);
  2633. for (i = 0; i < jstate->clocations_count; i++)
  2634. {
  2635. int off, /* Offset from start for cur tok */
  2636. tok_len; /* Length (in bytes) of that tok */
  2637. off = jstate->clocations[i].location;
  2638. /* Adjust recorded location if we're dealing with partial string */
  2639. off -= query_loc;
  2640. tok_len = jstate->clocations[i].length;
  2641. if (tok_len < 0)
  2642. continue; /* ignore any duplicates */
  2643. /* Copy next chunk (what precedes the next constant) */
  2644. len_to_wrt = off - last_off;
  2645. len_to_wrt -= last_tok_len;
  2646. Assert(len_to_wrt >= 0);
  2647. memcpy(norm_query + n_quer_loc, query + quer_loc, len_to_wrt);
  2648. n_quer_loc += len_to_wrt;
  2649. /* And insert a param symbol in place of the constant token */
  2650. n_quer_loc += sprintf(norm_query + n_quer_loc, "$%d",
  2651. i + 1 + jstate->highest_extern_param_id);
  2652. quer_loc = off + tok_len;
  2653. last_off = off;
  2654. last_tok_len = tok_len;
  2655. }
  2656. /*
  2657. * We've copied up until the last ignorable constant. Copy over the
  2658. * remaining bytes of the original query string.
  2659. */
  2660. len_to_wrt = query_len - quer_loc;
  2661. Assert(len_to_wrt >= 0);
  2662. memcpy(norm_query + n_quer_loc, query + quer_loc, len_to_wrt);
  2663. n_quer_loc += len_to_wrt;
  2664. Assert(n_quer_loc <= norm_query_buflen);
  2665. norm_query[n_quer_loc] = '\0';
  2666. *query_len_p = n_quer_loc;
  2667. return norm_query;
  2668. }
  2669. /*
  2670. * Given a valid SQL string and an array of constant-location records,
  2671. * fill in the textual lengths of those constants.
  2672. *
  2673. * The constants may use any allowed constant syntax, such as float literals,
  2674. * bit-strings, single-quoted strings and dollar-quoted strings. This is
  2675. * accomplished by using the public API for the core scanner.
  2676. *
  2677. * It is the caller's job to ensure that the string is a valid SQL statement
  2678. * with constants at the indicated locations. Since in practice the string
  2679. * has already been parsed, and the locations that the caller provides will
  2680. * have originated from within the authoritative parser, this should not be
  2681. * a problem.
  2682. *
  2683. * Duplicate constant pointers are possible, and will have their lengths
  2684. * marked as '-1', so that they are later ignored. (Actually, we assume the
  2685. * lengths were initialized as -1 to start with, and don't change them here.)
  2686. *
  2687. * If query_loc > 0, then "query" has been advanced by that much compared to
  2688. * the original string start, so we need to translate the provided locations
  2689. * to compensate. (This lets us avoid re-scanning statements before the one
  2690. * of interest, so it's worth doing.)
  2691. *
  2692. * N.B. There is an assumption that a '-' character at a Const location begins
  2693. * a negative numeric constant. This precludes there ever being another
  2694. * reason for a constant to start with a '-'.
  2695. */
  2696. static void
  2697. fill_in_constant_lengths(pgssJumbleState *jstate, const char *query,
  2698. int query_loc)
  2699. {
  2700. pgssLocationLen *locs;
  2701. core_yyscan_t yyscanner;
  2702. core_yy_extra_type yyextra;
  2703. core_YYSTYPE yylval;
  2704. YYLTYPE yylloc;
  2705. int last_loc = -1;
  2706. int i;
  2707. /*
  2708. * Sort the records by location so that we can process them in order while
  2709. * scanning the query text.
  2710. */
  2711. if (jstate->clocations_count > 1)
  2712. qsort(jstate->clocations, jstate->clocations_count,
  2713. sizeof(pgssLocationLen), comp_location);
  2714. locs = jstate->clocations;
  2715. /* initialize the flex scanner --- should match raw_parser() */
  2716. yyscanner = scanner_init(query,
  2717. &yyextra,
  2718. ScanKeywords,
  2719. NumScanKeywords);
  2720. /* we don't want to re-emit any escape string warnings */
  2721. yyextra.escape_string_warning = false;
  2722. /* Search for each constant, in sequence */
  2723. for (i = 0; i < jstate->clocations_count; i++)
  2724. {
  2725. int loc = locs[i].location;
  2726. int tok;
  2727. /* Adjust recorded location if we're dealing with partial string */
  2728. loc -= query_loc;
  2729. Assert(loc >= 0);
  2730. if (loc <= last_loc)
  2731. continue; /* Duplicate constant, ignore */
  2732. /* Lex tokens until we find the desired constant */
  2733. for (;;)
  2734. {
  2735. tok = core_yylex(&yylval, &yylloc, yyscanner);
  2736. /* We should not hit end-of-string, but if we do, behave sanely */
  2737. if (tok == 0)
  2738. break; /* out of inner for-loop */
  2739. /*
  2740. * We should find the token position exactly, but if we somehow
  2741. * run past it, work with that.
  2742. */
  2743. if (yylloc >= loc)
  2744. {
  2745. if (query[loc] == '-')
  2746. {
  2747. /*
  2748. * It's a negative value - this is the one and only case
  2749. * where we replace more than a single token.
  2750. *
  2751. * Do not compensate for the core system's special-case
  2752. * adjustment of location to that of the leading '-'
  2753. * operator in the event of a negative constant. It is
  2754. * also useful for our purposes to start from the minus
  2755. * symbol. In this way, queries like "select * from foo
  2756. * where bar = 1" and "select * from foo where bar = -2"
  2757. * will have identical normalized query strings.
  2758. */
  2759. tok = core_yylex(&yylval, &yylloc, yyscanner);
  2760. if (tok == 0)
  2761. break; /* out of inner for-loop */
  2762. }
  2763. /*
  2764. * We now rely on the assumption that flex has placed a zero
  2765. * byte after the text of the current token in scanbuf.
  2766. */
  2767. locs[i].length = strlen(yyextra.scanbuf + loc);
  2768. break; /* out of inner for-loop */
  2769. }
  2770. }
  2771. /* If we hit end-of-string, give up, leaving remaining lengths -1 */
  2772. if (tok == 0)
  2773. break;
  2774. last_loc = loc;
  2775. }
  2776. scanner_finish(yyscanner);
  2777. }
  2778. /*
  2779. * comp_location: comparator for qsorting pgssLocationLen structs by location
  2780. */
  2781. static int
  2782. comp_location(const void *a, const void *b)
  2783. {
  2784. int l = ((const pgssLocationLen *) a)->location;
  2785. int r = ((const pgssLocationLen *) b)->location;
  2786. if (l < r)
  2787. return -1;
  2788. else if (l > r)
  2789. return +1;
  2790. else
  2791. return 0;
  2792. }